diff --git a/api/api.go b/api/api.go index 33fcba409..5fa84931c 100644 --- a/api/api.go +++ b/api/api.go @@ -77,6 +77,8 @@ const ( ParamHeight = "height" ParamReference = "reference" ParamType = "type" + ParamSubtype = "subtype" + ParamSigner = "signer" ParamAccountIdFrom = "accountIdFrom" ParamAccountIdTo = "accountIdTo" ParamStartDateAfter = "startDateAfter" diff --git a/api/api_types.go b/api/api_types.go index efb390503..982e3c9d1 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -50,8 +50,11 @@ type AccountParams struct { // TransactionParams allows the client to filter transactions type TransactionParams struct { PaginationParams - Height uint64 `json:"height,omitempty"` - Type string `json:"type,omitempty"` + Hash string `json:"hash,omitempty"` + Height uint64 `json:"height,omitempty"` + Type string `json:"type,omitempty"` + Subtype string `json:"subtype,omitempty"` + Signer string `json:"signer,omitempty"` } // BlockParams allows the client to filter blocks @@ -275,8 +278,8 @@ type TransactionReference struct { // TransactionsList is used to return a paginated list to the client type TransactionsList struct { - Transactions []*indexertypes.Transaction `json:"transactions"` - Pagination *Pagination `json:"pagination"` + Transactions []*indexertypes.TransactionMetadata `json:"transactions"` + Pagination *Pagination `json:"pagination"` } // FeesList is used to return a paginated list to the client @@ -292,9 +295,8 @@ type TransfersList struct { } type GenericTransactionWithInfo struct { - TxContent json.RawMessage `json:"tx"` - TxInfo indexertypes.Transaction `json:"txInfo"` - Signature types.HexBytes `json:"signature"` + TxContent json.RawMessage `json:"tx"` + TxInfo *indexertypes.Transaction `json:"txInfo"` } type ChainInfo struct { @@ -444,8 +446,9 @@ func CensusTypeToOrigin(ctype CensusTypeDescription) (models.CensusOrigin, []byt } type Block struct { - comettypes.Block `json:",inline"` - Hash types.HexBytes `json:"hash" ` + comettypes.Header `json:"header"` + Hash types.HexBytes `json:"hash" ` + TxCount int64 `json:"txCount"` } // BlockList is used to return a paginated list to the client diff --git a/api/chain.go b/api/chain.go index 8c081bd89..d10865664 100644 --- a/api/chain.go +++ b/api/chain.go @@ -13,12 +13,9 @@ import ( "go.vocdoni.io/dvote/crypto/zk/circuit" "go.vocdoni.io/dvote/httprouter" "go.vocdoni.io/dvote/httprouter/apirest" - "go.vocdoni.io/dvote/types" "go.vocdoni.io/dvote/util" - "go.vocdoni.io/dvote/vochain" "go.vocdoni.io/dvote/vochain/genesis" "go.vocdoni.io/dvote/vochain/indexer" - "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/state" ) @@ -107,14 +104,6 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.Endpoint.RegisterMethod( - "/chain/transactions/reference/index/{index}", - "GET", - apirest.MethodAccessTypePublic, - a.chainTxRefByIndexHandler, - ); err != nil { - return err - } if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}/transactions/page/{page}", "GET", @@ -167,7 +156,7 @@ func (a *API) enableChainHandlers() error { "/chain/blocks/{height}", "GET", apirest.MethodAccessTypePublic, - a.chainBlockHandler, + a.chainBlockByHeightHandler, ); err != nil { return err } @@ -654,7 +643,7 @@ func (a *API) chainTxRefByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - ref, err := a.indexer.GetTxHashReference(hash) + ref, err := a.indexer.GetTxMetadataByHash(hash) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -690,15 +679,7 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er if err != nil { return err } - stx, err := a.vocapp.GetTx(uint32(height), int32(index)) - if err != nil { - if errors.Is(err, vochain.ErrTransactionNotFound) { - return ErrTransactionNotFound - } - return ErrVochainGetTxFailed.WithErr(err) - } - - ref, err := a.indexer.GetTxReferenceByBlockHeightAndBlockIndex(height, index) + ref, err := a.indexer.GetTransactionByHeightAndIndex(height, index) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -706,9 +687,8 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ErrVochainGetTxFailed.WithErr(err) } tx := &GenericTransactionWithInfo{ - TxContent: protoTxAsJSON(stx.Tx), - Signature: stx.Signature, - TxInfo: *ref, + TxContent: protoTxAsJSON(ref.RawTx), + TxInfo: ref, } data, err := json.Marshal(tx) if err != nil { @@ -717,36 +697,6 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ctx.Send(data, apirest.HTTPstatusOK) } -// chainTxRefByIndexHandler -// -// @Summary Transaction by index -// @Description Get transaction by its index. This is not transaction reference (hash), and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). -// @Tags Chain -// @Accept json -// @Produce json -// @Param index path int true "Index of the transaction" -// @Success 200 {object} indexertypes.Transaction -// @Success 204 "See [errors](vocdoni-api#errors) section" -// @Router /chain/transactions/reference/index/{index} [get] -func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - index, err := strconv.ParseUint(ctx.URLParam("index"), 10, 64) - if err != nil { - return err - } - ref, err := a.indexer.GetTransaction(index) - if err != nil { - if errors.Is(err, indexer.ErrTransactionNotFound) { - return ErrTransactionNotFound - } - return ErrVochainGetTxFailed.WithErr(err) - } - data, err := json.Marshal(ref) - if err != nil { - return err - } - return ctx.Send(data, apirest.HTTPstatusOK) -} - // chainTxListHandler // // @Summary List transactions @@ -758,14 +708,19 @@ func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC // @Param limit query number false "Items per page" // @Param height query number false "Block height" // @Param type query string false "Tx type" +// @Param subtype query string false "Tx subtype" +// @Param signer query string false "Tx signer" // @Success 200 {object} TransactionsList "List of transactions references" // @Router /chain/transactions [get] func (a *API) chainTxListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { params, err := parseTransactionParams( ctx.QueryParam(ParamPage), ctx.QueryParam(ParamLimit), + ctx.QueryParam(ParamHash), ctx.QueryParam(ParamHeight), ctx.QueryParam(ParamType), + ctx.QueryParam(ParamSubtype), + ctx.QueryParam(ParamSigner), ) if err != nil { return err @@ -797,6 +752,9 @@ func (a *API) chainTxListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTPC "", "", "", + "", + "", + "", ) if err != nil { return err @@ -833,6 +791,9 @@ func (a *API) chainTxListByHeightAndPageHandler(_ *apirest.APIdata, ctx *httprou "", ctx.URLParam(ParamHeight), "", + "", + "", + "", ) if err != nil { return err @@ -858,7 +819,10 @@ func (a *API) transactionList(params *TransactionParams) (*TransactionsList, err params.Limit, params.Page*params.Limit, params.Height, + params.Hash, params.Type, + params.Subtype, + params.Signer, ) if err != nil { return nil, ErrIndexerQueryFailed.WithErr(err) @@ -911,7 +875,7 @@ func (a *API) chainValidatorsHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCon return ctx.Send(data, apirest.HTTPstatusOK) } -// chainBlockHandler +// chainBlockByHeightHandler // // @Summary Get block (by height) // @Description Returns the full block information at the given height @@ -921,23 +885,34 @@ func (a *API) chainValidatorsHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCon // @Param height path int true "Block height" // @Success 200 {object} api.Block // @Router /chain/blocks/{height} [get] -func (a *API) chainBlockHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { +func (a *API) chainBlockByHeightHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { height, err := strconv.ParseUint(ctx.URLParam(ParamHeight), 10, 64) if err != nil { return err } - tmblock := a.vocapp.GetBlockByHeight(int64(height)) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHeight(int64(height)) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) + } + txcount, err := a.indexer.CountTransactionsByHeight(int64(height)) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ - Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, + TxCount: txcount, } data, err := json.Marshal(block) if err != nil { @@ -961,18 +936,29 @@ func (a *API) chainBlockByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - tmblock := a.vocapp.GetBlockByHash(hash) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHash(hash) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) + } + txcount, err := a.indexer.CountTransactionsByHeight(idxblock.Height) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ - Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, + TxCount: txcount, } data, err := json.Marshal(block) if err != nil { @@ -1015,39 +1001,7 @@ func (a *API) chainBlockListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCont // // Errors returned are always of type APIerror. func (a *API) sendBlockList(ctx *httprouter.HTTPContext, params *BlockParams) error { - // TODO: replace this by a.indexer.BlockList when it's available - blockList := func(limit, offset int, _, _, _ string) ([]*indexertypes.Block, uint64, error) { - if offset < 0 { - return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) - } - if limit <= 0 { - return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) - } - height := a.vocapp.Height() - total := uint64(height) - uint64(a.vocapp.Node.BlockStore().Base()) - start := height - uint32(params.Page*params.Limit) - end := start - uint32(params.Limit) - list := []*indexertypes.Block{} - for h := start; h > end; h-- { - tmblock := a.vocapp.GetBlockByHeight(int64(h)) - if tmblock == nil { - break - } - list = append(list, &indexertypes.Block{ - ChainID: tmblock.ChainID, - Height: tmblock.Height, - Time: tmblock.Time, - Hash: types.HexBytes(tmblock.Hash()), - ProposerAddress: tmblock.ProposerAddress.Bytes(), - LastBlockHash: tmblock.LastBlockID.Hash.Bytes(), - TxCount: int64(len(tmblock.Txs)), - }) - } - - return list, uint64(total), nil - } - - blocks, total, err := blockList( + blocks, total, err := a.indexer.BlockList( params.Limit, params.Page*params.Limit, params.ChainID, @@ -1401,7 +1355,7 @@ func parseTransfersParams(paramPage, paramLimit, paramAccountId, paramAccountIdF } // parseTransactionParams returns an TransactionParams filled with the passed params -func parseTransactionParams(paramPage, paramLimit, paramHeight, paramType string) (*TransactionParams, error) { +func parseTransactionParams(paramPage, paramLimit, paramHash, paramHeight, paramType, paramSubtype, paramSigner string) (*TransactionParams, error) { pagination, err := parsePaginationParams(paramPage, paramLimit) if err != nil { return nil, err @@ -1414,8 +1368,11 @@ func parseTransactionParams(paramPage, paramLimit, paramHeight, paramType string return &TransactionParams{ PaginationParams: pagination, + Hash: util.TrimHex(paramHash), Height: uint64(height), Type: paramType, + Subtype: paramSubtype, + Signer: util.TrimHex(paramSigner), }, nil } diff --git a/test/api_test.go b/test/api_test.go index 80e4f3699..267895636 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -461,6 +461,41 @@ func TestAPIAccountTokentxs(t *testing.T) { qt.Assert(t, gotAcct1.Balance, qt.Equals, initBalance+amountAcc2toAcct1-amountAcc1toAcct2-uint64(txBasePrice)) } +func TestAPIBlocks(t *testing.T) { + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.VoteHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + ) + token1 := uuid.New() + c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + // Block 1 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 1) + + // create a new account + initBalance := uint64(80) + _ = createAccount(t, c, server, initBalance) + + // Block 2 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 2) + + // check the txCount + resp, code := c.Request("GET", nil, "chain", "blocks", "1") + qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) + + block := api.Block{} + err := json.Unmarshal(resp, &block) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, block.TxCount, qt.Equals, int64(1)) +} + func runAPIElectionCostWithParams(t *testing.T, electionParams electionprice.ElectionParameters, startBlock uint32, initialBalance, diff --git a/vochain/app.go b/vochain/app.go index 7e0107537..f6691be61 100644 --- a/vochain/app.go +++ b/vochain/app.go @@ -290,10 +290,6 @@ func (app *BaseApplication) beginBlock(t time.Time, height uint32) { app.State.SetHeight(height) go app.State.CachePurge(height) - app.State.OnBeginBlock(vstate.BeginBlock{ - Height: int64(height), - Time: t, - }) } // endBlock is called at the end of every block. diff --git a/vochain/appsetup.go b/vochain/appsetup.go index fb52f66ed..6726b8b21 100644 --- a/vochain/appsetup.go +++ b/vochain/appsetup.go @@ -25,9 +25,6 @@ func (app *BaseApplication) SetNode(vochaincfg *config.VochainCfg) error { if app.Node, err = newTendermint(app, vochaincfg); err != nil { return fmt.Errorf("could not set tendermint node service: %s", err) } - if vochaincfg.IsSeedNode { - return nil - } // Note that cometcli.New logs any error rather than returning it. app.NodeClient = cometcli.New(app.Node) return nil diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 631e83205..7e289857a 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -85,6 +85,7 @@ func BenchmarkIndexer(b *testing.B) { tx := &vochaintx.Tx{ TxID: rnd.Random32(), TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, } idx.OnNewTx(tx, height, txBlockIndex) curTxs = append(curTxs, tx) @@ -112,7 +113,7 @@ func BenchmarkIndexer(b *testing.B) { qt.Check(b, bytes.Equal(voteRef.Meta.TxHash, tx.TxID[:]), qt.IsTrue) } - txRef, err := idx.GetTxHashReference(tx.TxID[:]) + txRef, err := idx.GetTxMetadataByHash(tx.TxID[:]) qt.Check(b, err, qt.IsNil) if err == nil { qt.Check(b, txRef.BlockHeight, qt.Equals, vote.Height) @@ -138,7 +139,11 @@ func BenchmarkFetchTx(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for j := 0; j < numTxs; j++ { - idx.OnNewTx(&vochaintx.Tx{TxID: util.Random32()}, uint32(i), int32(j)) + idx.OnNewTx(&vochaintx.Tx{ + TxID: util.Random32(), + TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, + }, uint32(i), int32(j)) } err := idx.Commit(uint32(i)) qt.Assert(b, err, qt.IsNil) @@ -147,14 +152,14 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTransaction(uint64((i * numTxs) + j + 1)) + _, err = idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(b, err, qt.IsNil) } - log.Infof("fetched %d transactions (out of %d total) by index, took %s", + log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxHashReference([]byte(fmt.Sprintf("hash%d%d", i, j))) + _, err = idx.GetTxMetadataByHash([]byte(fmt.Sprintf("hash%d%d", i, j))) qt.Assert(b, err, qt.IsNil) } log.Infof("fetched %d transactions (out of %d total) by hash, took %s", diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index e40a5fdfb..c6021e530 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -7,35 +7,86 @@ import ( "fmt" "time" - "go.vocdoni.io/dvote/log" indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" - "go.vocdoni.io/dvote/vochain/state" + "go.vocdoni.io/dvote/vochain/indexer/indexertypes" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. var ErrBlockNotFound = fmt.Errorf("block not found") -func (idx *Indexer) OnBeginBlock(bb state.BeginBlock) { - idx.blockMu.Lock() - defer idx.blockMu.Unlock() - queries := idx.blockTxQueries() - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - }); err != nil { - log.Errorw(err, "cannot index new block") +// BlockTimestamp returns the timestamp of the block at the given height +func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { + block, err := idx.BlockByHeight(height) + if err != nil { + return time.Time{}, err } + return block.Time, nil } -// BlockTimestamp returns the timestamp of the block at the given height -func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { - block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) +// BlockByHeight returns the available information of the block at the given height +func (idx *Indexer) BlockByHeight(height int64) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), height) if err != nil { if errors.Is(err, sql.ErrNoRows) { - return time.Time{}, ErrBlockNotFound + return nil, ErrBlockNotFound } - return time.Time{}, err + return nil, err } - return block.Time, nil + return indexertypes.BlockFromDB(&block), nil +} + +// BlockByHash returns the available information of the block with the given hash +func (idx *Indexer) BlockByHash(hash []byte) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHash(context.TODO(), hash) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockList returns the list of blocks indexed. +// chainID, hash, proposerAddress are optional, if declared as zero-value will be ignored. +// The first one returned is the newest, so they are in descending order. +func (idx *Indexer) BlockList(limit, offset int, chainID, hash, proposerAddress string) ([]*indexertypes.Block, uint64, error) { + if offset < 0 { + return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) + } + if limit <= 0 { + return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) + } + results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ + Limit: int64(limit), + Offset: int64(offset), + ChainID: chainID, + HashSubstr: hash, + ProposerAddress: proposerAddress, + }) + if err != nil { + return nil, 0, err + } + list := []*indexertypes.Block{} + for _, row := range results { + list = append(list, indexertypes.BlockFromDBRow(&row)) + } + if len(results) == 0 { + return list, 0, nil + } + return list, uint64(results[0].TotalCount), nil +} + +// CountBlocks returns how many blocks are indexed. +func (idx *Indexer) CountBlocks() (uint64, error) { + results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ + Limit: 1, + }) + if err != nil { + return 0, err + } + if len(results) == 0 { + return 0, nil + } + return uint64(results[0].TotalCount), nil } diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index ffa3fa896..669f59602 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -13,31 +13,156 @@ import ( const createBlock = `-- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ) +ON CONFLICT(height) DO UPDATE +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash ` type CreateBlockParams struct { - Height int64 - Time time.Time - DataHash []byte + ChainID string + Height int64 + Time time.Time + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } func (q *Queries) CreateBlock(ctx context.Context, arg CreateBlockParams) (sql.Result, error) { - return q.exec(ctx, q.createBlockStmt, createBlock, arg.Height, arg.Time, arg.DataHash) + return q.exec(ctx, q.createBlockStmt, createBlock, + arg.ChainID, + arg.Height, + arg.Time, + arg.Hash, + arg.ProposerAddress, + arg.LastBlockHash, + ) } -const getBlock = `-- name: GetBlock :one -SELECT height, time, data_hash FROM blocks +const getBlockByHash = `-- name: GetBlockByHash :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks +WHERE hash = ? +LIMIT 1 +` + +func (q *Queries) GetBlockByHash(ctx context.Context, hash []byte) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHashStmt, getBlockByHash, hash) + var i Block + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) + return i, err +} + +const getBlockByHeight = `-- name: GetBlockByHeight :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks WHERE height = ? LIMIT 1 ` -func (q *Queries) GetBlock(ctx context.Context, height int64) (Block, error) { - row := q.queryRow(ctx, q.getBlockStmt, getBlock, height) +func (q *Queries) GetBlockByHeight(ctx context.Context, height int64) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHeightStmt, getBlockByHeight, height) var i Block - err := row.Scan(&i.Height, &i.Time, &i.DataHash) + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) return i, err } + +const searchBlocks = `-- name: SearchBlocks :many +SELECT + b.height, b.time, b.chain_id, b.hash, b.proposer_address, b.last_block_hash, + COUNT(t.block_index) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (?1 = '' OR b.chain_id = ?1) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(b.hash)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(?2)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (?3 = '' OR LOWER(HEX(b.proposer_address)) = LOWER(?3)) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT ?5 +OFFSET ?4 +` + +type SearchBlocksParams struct { + ChainID interface{} + HashSubstr interface{} + ProposerAddress interface{} + Offset int64 + Limit int64 +} + +type SearchBlocksRow struct { + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte + TxCount int64 + TotalCount int64 +} + +func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) { + rows, err := q.query(ctx, q.searchBlocksStmt, searchBlocks, + arg.ChainID, + arg.HashSubstr, + arg.ProposerAddress, + arg.Offset, + arg.Limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchBlocksRow + for rows.Next() { + var i SearchBlocksRow + if err := rows.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + &i.TxCount, + &i.TotalCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 3695e8b73..47bf88d46 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -36,6 +36,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.countTransactionsStmt, err = db.PrepareContext(ctx, countTransactions); err != nil { return nil, fmt.Errorf("error preparing query CountTransactions: %w", err) } + if q.countTransactionsByHeightStmt, err = db.PrepareContext(ctx, countTransactionsByHeight); err != nil { + return nil, fmt.Errorf("error preparing query CountTransactionsByHeight: %w", err) + } if q.countVotesStmt, err = db.PrepareContext(ctx, countVotes); err != nil { return nil, fmt.Errorf("error preparing query CountVotes: %w", err) } @@ -60,8 +63,11 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.createVoteStmt, err = db.PrepareContext(ctx, createVote); err != nil { return nil, fmt.Errorf("error preparing query CreateVote: %w", err) } - if q.getBlockStmt, err = db.PrepareContext(ctx, getBlock); err != nil { - return nil, fmt.Errorf("error preparing query GetBlock: %w", err) + if q.getBlockByHashStmt, err = db.PrepareContext(ctx, getBlockByHash); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHash: %w", err) + } + if q.getBlockByHeightStmt, err = db.PrepareContext(ctx, getBlockByHeight); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHeight: %w", err) } if q.getEntityCountStmt, err = db.PrepareContext(ctx, getEntityCount); err != nil { return nil, fmt.Errorf("error preparing query GetEntityCount: %w", err) @@ -81,14 +87,11 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTokenTransferStmt, err = db.PrepareContext(ctx, getTokenTransfer); err != nil { return nil, fmt.Errorf("error preparing query GetTokenTransfer: %w", err) } - if q.getTransactionStmt, err = db.PrepareContext(ctx, getTransaction); err != nil { - return nil, fmt.Errorf("error preparing query GetTransaction: %w", err) - } if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt, err = db.PrepareContext(ctx, getTxReferenceByBlockHeightAndBlockIndex); err != nil { - return nil, fmt.Errorf("error preparing query GetTxReferenceByBlockHeightAndBlockIndex: %w", err) + if q.getTransactionByHeightAndIndexStmt, err = db.PrepareContext(ctx, getTransactionByHeightAndIndex); err != nil { + return nil, fmt.Errorf("error preparing query GetTransactionByHeightAndIndex: %w", err) } if q.getVoteStmt, err = db.PrepareContext(ctx, getVote); err != nil { return nil, fmt.Errorf("error preparing query GetVote: %w", err) @@ -96,6 +99,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.searchAccountsStmt, err = db.PrepareContext(ctx, searchAccounts); err != nil { return nil, fmt.Errorf("error preparing query SearchAccounts: %w", err) } + if q.searchBlocksStmt, err = db.PrepareContext(ctx, searchBlocks); err != nil { + return nil, fmt.Errorf("error preparing query SearchBlocks: %w", err) + } if q.searchEntitiesStmt, err = db.PrepareContext(ctx, searchEntities); err != nil { return nil, fmt.Errorf("error preparing query SearchEntities: %w", err) } @@ -157,6 +163,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing countTransactionsStmt: %w", cerr) } } + if q.countTransactionsByHeightStmt != nil { + if cerr := q.countTransactionsByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countTransactionsByHeightStmt: %w", cerr) + } + } if q.countVotesStmt != nil { if cerr := q.countVotesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing countVotesStmt: %w", cerr) @@ -197,9 +208,14 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing createVoteStmt: %w", cerr) } } - if q.getBlockStmt != nil { - if cerr := q.getBlockStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getBlockStmt: %w", cerr) + if q.getBlockByHashStmt != nil { + if cerr := q.getBlockByHashStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHashStmt: %w", cerr) + } + } + if q.getBlockByHeightStmt != nil { + if cerr := q.getBlockByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHeightStmt: %w", cerr) } } if q.getEntityCountStmt != nil { @@ -232,19 +248,14 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTokenTransferStmt: %w", cerr) } } - if q.getTransactionStmt != nil { - if cerr := q.getTransactionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTransactionStmt: %w", cerr) - } - } if q.getTransactionByHashStmt != nil { if cerr := q.getTransactionByHashStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) } } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt != nil { - if cerr := q.getTxReferenceByBlockHeightAndBlockIndexStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTxReferenceByBlockHeightAndBlockIndexStmt: %w", cerr) + if q.getTransactionByHeightAndIndexStmt != nil { + if cerr := q.getTransactionByHeightAndIndexStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getTransactionByHeightAndIndexStmt: %w", cerr) } } if q.getVoteStmt != nil { @@ -257,6 +268,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing searchAccountsStmt: %w", cerr) } } + if q.searchBlocksStmt != nil { + if cerr := q.searchBlocksStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchBlocksStmt: %w", cerr) + } + } if q.searchEntitiesStmt != nil { if cerr := q.searchEntitiesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchEntitiesStmt: %w", cerr) @@ -354,85 +370,89 @@ func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, ar } type Queries struct { - db DBTX - tx *sql.Tx - computeProcessVoteCountStmt *sql.Stmt - countAccountsStmt *sql.Stmt - countTokenTransfersByAccountStmt *sql.Stmt - countTransactionsStmt *sql.Stmt - countVotesStmt *sql.Stmt - createAccountStmt *sql.Stmt - createBlockStmt *sql.Stmt - createProcessStmt *sql.Stmt - createTokenFeeStmt *sql.Stmt - createTokenTransferStmt *sql.Stmt - createTransactionStmt *sql.Stmt - createVoteStmt *sql.Stmt - getBlockStmt *sql.Stmt - getEntityCountStmt *sql.Stmt - getProcessStmt *sql.Stmt - getProcessCountStmt *sql.Stmt - getProcessIDsByFinalResultsStmt *sql.Stmt - getProcessStatusStmt *sql.Stmt - getTokenTransferStmt *sql.Stmt - getTransactionStmt *sql.Stmt - getTransactionByHashStmt *sql.Stmt - getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt - getVoteStmt *sql.Stmt - searchAccountsStmt *sql.Stmt - searchEntitiesStmt *sql.Stmt - searchProcessesStmt *sql.Stmt - searchTokenFeesStmt *sql.Stmt - searchTokenTransfersStmt *sql.Stmt - searchTransactionsStmt *sql.Stmt - searchVotesStmt *sql.Stmt - setProcessResultsCancelledStmt *sql.Stmt - setProcessResultsReadyStmt *sql.Stmt - updateProcessEndDateStmt *sql.Stmt - updateProcessFromStateStmt *sql.Stmt - updateProcessResultByIDStmt *sql.Stmt - updateProcessResultsStmt *sql.Stmt + db DBTX + tx *sql.Tx + computeProcessVoteCountStmt *sql.Stmt + countAccountsStmt *sql.Stmt + countTokenTransfersByAccountStmt *sql.Stmt + countTransactionsStmt *sql.Stmt + countTransactionsByHeightStmt *sql.Stmt + countVotesStmt *sql.Stmt + createAccountStmt *sql.Stmt + createBlockStmt *sql.Stmt + createProcessStmt *sql.Stmt + createTokenFeeStmt *sql.Stmt + createTokenTransferStmt *sql.Stmt + createTransactionStmt *sql.Stmt + createVoteStmt *sql.Stmt + getBlockByHashStmt *sql.Stmt + getBlockByHeightStmt *sql.Stmt + getEntityCountStmt *sql.Stmt + getProcessStmt *sql.Stmt + getProcessCountStmt *sql.Stmt + getProcessIDsByFinalResultsStmt *sql.Stmt + getProcessStatusStmt *sql.Stmt + getTokenTransferStmt *sql.Stmt + getTransactionByHashStmt *sql.Stmt + getTransactionByHeightAndIndexStmt *sql.Stmt + getVoteStmt *sql.Stmt + searchAccountsStmt *sql.Stmt + searchBlocksStmt *sql.Stmt + searchEntitiesStmt *sql.Stmt + searchProcessesStmt *sql.Stmt + searchTokenFeesStmt *sql.Stmt + searchTokenTransfersStmt *sql.Stmt + searchTransactionsStmt *sql.Stmt + searchVotesStmt *sql.Stmt + setProcessResultsCancelledStmt *sql.Stmt + setProcessResultsReadyStmt *sql.Stmt + updateProcessEndDateStmt *sql.Stmt + updateProcessFromStateStmt *sql.Stmt + updateProcessResultByIDStmt *sql.Stmt + updateProcessResultsStmt *sql.Stmt } func (q *Queries) WithTx(tx *sql.Tx) *Queries { return &Queries{ - db: tx, - tx: tx, - computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, - countAccountsStmt: q.countAccountsStmt, - countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, - countTransactionsStmt: q.countTransactionsStmt, - countVotesStmt: q.countVotesStmt, - createAccountStmt: q.createAccountStmt, - createBlockStmt: q.createBlockStmt, - createProcessStmt: q.createProcessStmt, - createTokenFeeStmt: q.createTokenFeeStmt, - createTokenTransferStmt: q.createTokenTransferStmt, - createTransactionStmt: q.createTransactionStmt, - createVoteStmt: q.createVoteStmt, - getBlockStmt: q.getBlockStmt, - getEntityCountStmt: q.getEntityCountStmt, - getProcessStmt: q.getProcessStmt, - getProcessCountStmt: q.getProcessCountStmt, - getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, - getProcessStatusStmt: q.getProcessStatusStmt, - getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionStmt: q.getTransactionStmt, - getTransactionByHashStmt: q.getTransactionByHashStmt, - getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, - getVoteStmt: q.getVoteStmt, - searchAccountsStmt: q.searchAccountsStmt, - searchEntitiesStmt: q.searchEntitiesStmt, - searchProcessesStmt: q.searchProcessesStmt, - searchTokenFeesStmt: q.searchTokenFeesStmt, - searchTokenTransfersStmt: q.searchTokenTransfersStmt, - searchTransactionsStmt: q.searchTransactionsStmt, - searchVotesStmt: q.searchVotesStmt, - setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, - setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, - updateProcessEndDateStmt: q.updateProcessEndDateStmt, - updateProcessFromStateStmt: q.updateProcessFromStateStmt, - updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, - updateProcessResultsStmt: q.updateProcessResultsStmt, + db: tx, + tx: tx, + computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, + countAccountsStmt: q.countAccountsStmt, + countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, + countTransactionsStmt: q.countTransactionsStmt, + countTransactionsByHeightStmt: q.countTransactionsByHeightStmt, + countVotesStmt: q.countVotesStmt, + createAccountStmt: q.createAccountStmt, + createBlockStmt: q.createBlockStmt, + createProcessStmt: q.createProcessStmt, + createTokenFeeStmt: q.createTokenFeeStmt, + createTokenTransferStmt: q.createTokenTransferStmt, + createTransactionStmt: q.createTransactionStmt, + createVoteStmt: q.createVoteStmt, + getBlockByHashStmt: q.getBlockByHashStmt, + getBlockByHeightStmt: q.getBlockByHeightStmt, + getEntityCountStmt: q.getEntityCountStmt, + getProcessStmt: q.getProcessStmt, + getProcessCountStmt: q.getProcessCountStmt, + getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, + getProcessStatusStmt: q.getProcessStatusStmt, + getTokenTransferStmt: q.getTokenTransferStmt, + getTransactionByHashStmt: q.getTransactionByHashStmt, + getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, + getVoteStmt: q.getVoteStmt, + searchAccountsStmt: q.searchAccountsStmt, + searchBlocksStmt: q.searchBlocksStmt, + searchEntitiesStmt: q.searchEntitiesStmt, + searchProcessesStmt: q.searchProcessesStmt, + searchTokenFeesStmt: q.searchTokenFeesStmt, + searchTokenTransfersStmt: q.searchTokenTransfersStmt, + searchTransactionsStmt: q.searchTransactionsStmt, + searchVotesStmt: q.searchVotesStmt, + setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, + setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, + updateProcessEndDateStmt: q.updateProcessEndDateStmt, + updateProcessFromStateStmt: q.updateProcessFromStateStmt, + updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, + updateProcessResultsStmt: q.updateProcessResultsStmt, } } diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 08a6e0e2b..d69facf2f 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -11,9 +11,12 @@ import ( ) type Block struct { - Height int64 - Time time.Time - DataHash []byte + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } type Process struct { @@ -57,9 +60,12 @@ type TokenTransfer struct { } type Transaction struct { - ID int64 Hash types.Hash BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index 2d06f135a..7fabaa528 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -23,12 +23,32 @@ func (q *Queries) CountTransactions(ctx context.Context) (int64, error) { return count, err } +const countTransactionsByHeight = `-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ? +` + +func (q *Queries) CountTransactionsByHeight(ctx context.Context, blockHeight int64) (int64, error) { + row := q.queryRow(ctx, q.countTransactionsByHeightStmt, countTransactionsByHeight, blockHeight) + var count int64 + err := row.Scan(&count) + return count, err +} + const createTransaction = `-- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ) +ON CONFLICT(hash) DO UPDATE +SET block_height = excluded.block_height, + block_index = excluded.block_index, + type = excluded.type, + subtype = excluded.subtype, + raw_tx = excluded.raw_tx, + signature = excluded.signature, + signer = excluded.signer ` type CreateTransactionParams struct { @@ -36,6 +56,10 @@ type CreateTransactionParams struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (sql.Result, error) { @@ -44,30 +68,15 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.BlockHeight, arg.BlockIndex, arg.Type, + arg.Subtype, + arg.RawTx, + arg.Signature, + arg.Signer, ) } -const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type FROM transactions -WHERE id = ? -LIMIT 1 -` - -func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, error) { - row := q.queryRow(ctx, q.getTransactionStmt, getTransaction, id) - var i Transaction - err := row.Scan( - &i.ID, - &i.Hash, - &i.BlockHeight, - &i.BlockIndex, - &i.Type, - ) - return i, err -} - const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type FROM transactions +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE hash = ? LIMIT 1 ` @@ -76,77 +85,95 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr row := q.queryRow(ctx, q.getTransactionByHashStmt, getTransactionByHash, hash) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, ) return i, err } -const getTxReferenceByBlockHeightAndBlockIndex = `-- name: GetTxReferenceByBlockHeightAndBlockIndex :one -SELECT id, hash, block_height, block_index, type FROM transactions +const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` -type GetTxReferenceByBlockHeightAndBlockIndexParams struct { +type GetTransactionByHeightAndIndexParams struct { BlockHeight int64 BlockIndex int64 } -func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, arg GetTxReferenceByBlockHeightAndBlockIndexParams) (Transaction, error) { - row := q.queryRow(ctx, q.getTxReferenceByBlockHeightAndBlockIndexStmt, getTxReferenceByBlockHeightAndBlockIndex, arg.BlockHeight, arg.BlockIndex) +func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTransactionByHeightAndIndexParams) (Transaction, error) { + row := q.queryRow(ctx, q.getTransactionByHeightAndIndexStmt, getTransactionByHeightAndIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, ) return i, err } const searchTransactions = `-- name: SearchTransactions :many -WITH results AS ( - SELECT id, hash, block_height, block_index, type - FROM transactions - WHERE ( - (?3 = 0 OR block_height = ?3) - AND (?4 = '' OR LOWER(type) = LOWER(?4)) +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer, COUNT(*) OVER() AS total_count +FROM transactions +WHERE + (?1 = 0 OR block_height = ?1) + AND (?2 = '' OR LOWER(type) = LOWER(?2)) + AND (?3 = '' OR LOWER(subtype) = LOWER(?3)) + AND (?4 = '' OR LOWER(HEX(signer)) = LOWER(?4)) + AND ( + ?5 = '' + OR (LENGTH(?5) = 64 AND LOWER(HEX(hash)) = LOWER(?5)) + OR (LENGTH(?5) < 64 AND INSTR(LOWER(HEX(hash)), LOWER(?5)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches ) -) -SELECT id, hash, block_height, block_index, type, COUNT(*) OVER() AS total_count -FROM results -ORDER BY id DESC -LIMIT ?2 -OFFSET ?1 +ORDER BY block_height DESC, block_index DESC +LIMIT ?7 +OFFSET ?6 ` type SearchTransactionsParams struct { - Offset int64 - Limit int64 BlockHeight interface{} TxType interface{} + TxSubtype interface{} + TxSigner interface{} + HashSubstr interface{} + Offset int64 + Limit int64 } type SearchTransactionsRow struct { - ID int64 - Hash []byte + Hash types.Hash BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte TotalCount int64 } func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactionsParams) ([]SearchTransactionsRow, error) { rows, err := q.query(ctx, q.searchTransactionsStmt, searchTransactions, - arg.Offset, - arg.Limit, arg.BlockHeight, arg.TxType, + arg.TxSubtype, + arg.TxSigner, + arg.HashSubstr, + arg.Offset, + arg.Limit, ) if err != nil { return nil, err @@ -156,11 +183,14 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions for rows.Next() { var i SearchTransactionsRow if err := rows.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, &i.TotalCount, ); err != nil { return nil, err diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 43b59323e..b764c8812 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -1,6 +1,7 @@ package indexer import ( + "bytes" "context" "database/sql" "embed" @@ -172,6 +173,12 @@ func (idx *Indexer) startDB() error { } goose.SetLogger(log.GooseLogger()) goose.SetBaseFS(embedMigrations) + + if gooseMigrationsPending(idx.readWriteDB, "migrations") { + log.Info("indexer db needs migration, scheduling a reindex after sync") + go idx.ReindexBlocks(false) + } + if err := goose.Up(idx.readWriteDB, "migrations"); err != nil { return fmt.Errorf("goose up: %w", err) } @@ -249,6 +256,27 @@ func (idx *Indexer) RestoreBackup(path string) error { return nil } +func gooseMigrationsPending(db *sql.DB, dir string) bool { + // Get the latest applied migration version + currentVersion, err := goose.GetDBVersion(db) + if err != nil { + log.Errorf("failed to get current database version: %v", err) + return false + } + + // Collect migrations after the current version + migrations, err := goose.CollectMigrations(dir, currentVersion, goose.MaxVersion) + if err != nil { + if errors.Is(err, goose.ErrNoMigrationFiles) { + return false + } + log.Errorf("failed to collect migrations: %v", err) + return false + } + + return len(migrations) > 0 +} + // SaveBackup backs up the database to a file on disk. // Note that writes to the database may be blocked until the backup finishes, // and an error may occur if a file at path already exists. @@ -402,6 +430,80 @@ func (idx *Indexer) AfterSyncBootstrap(inTest bool) { log.Infof("live results recovery computation finished, took %s", time.Since(startTime)) } +// ReindexBlocks reindexes all blocks found in blockstore +func (idx *Indexer) ReindexBlocks(inTest bool) { + if !inTest { + <-idx.App.WaitUntilSynced() + } + + // Note that holding blockMu means new votes aren't added until the reindex finishes. + idx.blockMu.Lock() + defer idx.blockMu.Unlock() + + if idx.App.Node == nil || idx.App.Node.BlockStore() == nil { + return + } + + idxBlockCount, err := idx.CountBlocks() + if err != nil { + log.Warnf("indexer CountBlocks returned error: %s", err) + } + log.Infow("start reindexing", + "blockStoreBase", idx.App.Node.BlockStore().Base(), + "blockStoreHeight", idx.App.Node.BlockStore().Height(), + "indexerBlockCount", idxBlockCount, + ) + queries := idx.blockTxQueries() + for height := idx.App.Node.BlockStore().Base(); height <= idx.App.Node.BlockStore().Height(); height++ { + if b := idx.App.GetBlockByHeight(int64(height)); b != nil { + // Blocks + func() { + idxBlock, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), b.Height) + if err == nil && idxBlock.Time != b.Time { + log.Errorf("while reindexing blocks, block %d timestamp in db (%s) differs from blockstore (%s), leaving untouched", height, idxBlock.Time, b.Time) + return + } + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } + }() + + // Transactions + func() { + for index, tx := range b.Data.Txs { + idxTx, err := idx.readOnlyQuery.GetTransactionByHeightAndIndex(context.TODO(), indexerdb.GetTransactionByHeightAndIndexParams{ + BlockHeight: b.Height, + BlockIndex: int64(index), + }) + if err == nil && !bytes.Equal(idxTx.Hash, tx.Hash()) { + log.Errorf("while reindexing txs, tx %d/%d hash in db (%x) differs from blockstore (%x), leaving untouched", b.Height, index, idxTx.Hash, tx.Hash()) + return + } + vtx := new(vochaintx.Tx) + if err := vtx.Unmarshal(tx, b.ChainID); err != nil { + log.Errorw(err, fmt.Sprintf("cannot unmarshal tx %d/%d", b.Height, index)) + continue + } + idx.indexTx(vtx, uint32(b.Height), int32(index)) + } + }() + } + } + + log.Infow("finished reindexing", + "blockStoreBase", idx.App.Node.BlockStore().Base(), + "blockStoreHeight", idx.App.Node.BlockStore().Height(), + "indexerBlockCount", idxBlockCount, + ) +} + // Commit is called by the APP when a block is confirmed and included into the chain func (idx *Indexer) Commit(height uint32) error { idx.blockMu.Lock() @@ -414,6 +516,20 @@ func (idx *Indexer) Commit(height uint32) error { queries := idx.blockTxQueries() ctx := context.TODO() + // index the new block + if b := idx.App.GetBlockByHeight(int64(height)); b != nil { + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } + } + for _, pidStr := range updateProcs { pid := types.ProcessID(pidStr) if err := idx.updateProcess(ctx, queries, pid); err != nil { diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index fad19d248..f3dc00ed3 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1392,6 +1392,7 @@ func TestTxIndexer(t *testing.T) { idx.OnNewTx(&vochaintx.Tx{ TxID: getTxID(i, j), TxModelType: "setAccount", + Tx: &models.Tx{Payload: &models.Tx_SetAccount{}}, }, uint32(i), int32(j)) } } @@ -1404,7 +1405,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTransaction(uint64(i*txsPerBlock + j + 1)) + ref, err := idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1412,28 +1413,25 @@ func TestTxIndexer(t *testing.T) { h := make([]byte, 32) id := getTxID(i, j) copy(h, id[:]) - hashRef, err := idx.GetTxHashReference(h) + hashRef, err := idx.GetTxMetadataByHash(h) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hashRef.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, hashRef.TxBlockIndex, qt.Equals, int32(j)) } } - txs, _, err := idx.SearchTransactions(15, 0, 0, "") + txs, _, err := idx.SearchTransactions(15, 0, 0, "", "", "", "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { - // Index is between 1 and totalCount. - qt.Assert(t, tx.Index, qt.Equals, uint64(totalTxs-i)) // BlockIndex and TxBlockIndex start at 0, so subtract 1. qt.Assert(t, tx.BlockHeight, qt.Equals, uint32(totalTxs-i-1)/txsPerBlock) qt.Assert(t, tx.TxBlockIndex, qt.Equals, int32(totalTxs-i-1)%txsPerBlock) qt.Assert(t, tx.TxType, qt.Equals, "setAccount") } - txs, _, err = idx.SearchTransactions(1, 5, 0, "") + txs, _, err = idx.SearchTransactions(1, 5, 0, "", "", "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) - qt.Assert(t, txs[0].Index, qt.Equals, uint64(95)) } func TestCensusUpdate(t *testing.T) { diff --git a/vochain/indexer/indexertypes/block.go b/vochain/indexer/indexertypes/block.go index 4954dbc22..4b711bdbc 100644 --- a/vochain/indexer/indexertypes/block.go +++ b/vochain/indexer/indexertypes/block.go @@ -4,6 +4,7 @@ import ( "time" "go.vocdoni.io/dvote/types" + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" ) // Block represents a block handled by the Vochain. @@ -18,3 +19,28 @@ type Block struct { LastBlockHash types.HexBytes `json:"lastBlockHash"` TxCount int64 `json:"txCount"` } + +// BlockFromDB converts the indexerdb.Block into a Block +func BlockFromDB(dbblock *indexerdb.Block) *Block { + return &Block{ + ChainID: dbblock.ChainID, + Height: dbblock.Height, + Time: dbblock.Time, + Hash: nonEmptyBytes(dbblock.Hash), + ProposerAddress: nonEmptyBytes(dbblock.ProposerAddress), + LastBlockHash: nonEmptyBytes(dbblock.LastBlockHash), + } +} + +// BlockFromDBRow converts the indexerdb.SearchBlocksRow into a Block +func BlockFromDBRow(row *indexerdb.SearchBlocksRow) *Block { + return &Block{ + ChainID: row.ChainID, + Height: row.Height, + Time: row.Time, + Hash: nonEmptyBytes(row.Hash), + ProposerAddress: nonEmptyBytes(row.ProposerAddress), + LastBlockHash: nonEmptyBytes(row.LastBlockHash), + TxCount: row.TxCount, + } +} diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index d0d2b29aa..7e54e25a7 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -176,30 +176,51 @@ type TxPackage struct { Signature types.HexBytes `json:"signature"` } -// TxMetadata contains tx information for the TransactionList api -type TxMetadata struct { - Type string `json:"type"` - BlockHeight uint32 `json:"blockHeight,omitempty"` - Index int32 `json:"index"` - Hash types.HexBytes `json:"hash"` +// TransactionMetadata contains tx information for the TransactionList api +type TransactionMetadata struct { + Hash types.HexBytes `json:"hash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` + BlockHeight uint32 `json:"height" format:"int32" example:"64924"` + TxBlockIndex int32 `json:"index" format:"int32" example:"0"` + TxType string `json:"type" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` + TxSubtype string `json:"subtype" example:"set_process_census"` + Signer types.HexBytes `json:"signer" swaggertype:"string" example:"0e45513942cf95330fc5e9020851b8bdd9b9c9df"` } -// Transaction holds the db reference for a single transaction -type Transaction struct { - Index uint64 `json:"transactionNumber" format:"int64" example:"944"` - Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` - BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` - TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` - TxType string `json:"transactionType" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` +func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { + return &TransactionMetadata{ + Hash: dbtx.Hash, + BlockHeight: uint32(dbtx.BlockHeight), + TxBlockIndex: int32(dbtx.BlockIndex), + TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, + Signer: dbtx.Signer, + } } -func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { - return &Transaction{ - Index: uint64(dbtx.ID), +func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { + return &TransactionMetadata{ Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, + Signer: dbtx.Signer, + } +} + +// Transaction holds a single transaction +type Transaction struct { + *TransactionMetadata + RawTx types.HexBytes `json:"-"` + Signature types.HexBytes `json:"signature,omitempty"` +} + +// TransactionFromDB converts an indexerdb.Transaction into a Transaction +func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { + return &Transaction{ + TransactionMetadata: TransactionMetadataFromDB(dbtx), + RawTx: dbtx.RawTx, + Signature: dbtx.Signature, } } diff --git a/vochain/indexer/migrations/0013_recreate_table_transactions.sql b/vochain/indexer/migrations/0013_recreate_table_transactions.sql new file mode 100644 index 000000000..bf091d7c9 --- /dev/null +++ b/vochain/indexer/migrations/0013_recreate_table_transactions.sql @@ -0,0 +1,56 @@ +-- +goose Up +PRAGMA foreign_keys = OFF; + +-- Create a new table with hash as primary key +CREATE TABLE transactions_new ( + hash BLOB NOT NULL PRIMARY KEY, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data from the old table to the new table +INSERT INTO transactions_new (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions; + +-- Drop the old table +DROP TABLE transactions; + +-- Rename the new table to the old table name +ALTER TABLE transactions_new RENAME TO transactions; + +-- Recreate necessary indexes +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; + +-- +goose Down +PRAGMA foreign_keys = OFF; + +-- Recreate the old table structure +CREATE TABLE transactions ( + id INTEGER NOT NULL PRIMARY KEY, + hash BLOB NOT NULL, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data back from the new table to the old table +INSERT INTO transactions (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions_new; + +-- Drop the new table +DROP TABLE transactions_new; + +-- Recreate the old indexes +CREATE INDEX transactions_hash +ON transactions(hash); + +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; diff --git a/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql b/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql new file mode 100644 index 000000000..c83c32c06 --- /dev/null +++ b/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE blocks DROP COLUMN data_hash; +ALTER TABLE blocks ADD COLUMN chain_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE blocks ADD COLUMN hash BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN proposer_address BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN last_block_hash BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE blocks ADD COLUMN data_hash BLOB NOT NULL; +ALTER TABLE blocks DROP COLUMN chain_id; +ALTER TABLE blocks DROP COLUMN hash; +ALTER TABLE blocks DROP COLUMN proposer_address; +ALTER TABLE blocks DROP COLUMN last_block_hash; diff --git a/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql b/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql new file mode 100644 index 000000000..81dcb9dec --- /dev/null +++ b/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql @@ -0,0 +1,11 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN subtype TEXT NOT NULL DEFAULT ''; +ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signature BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signer BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN signer; +ALTER TABLE transactions DROP COLUMN signature; +ALTER TABLE transactions DROP COLUMN raw_tx; +ALTER TABLE transactions DROP COLUMN subtype; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index 577e875b5..d15332cdf 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -1,11 +1,46 @@ -- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? -); + ?, ?, ?, ?, ?, ? +) +ON CONFLICT(height) DO UPDATE +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash; --- name: GetBlock :one +-- name: GetBlockByHeight :one SELECT * FROM blocks WHERE height = ? LIMIT 1; + +-- name: GetBlockByHash :one +SELECT * FROM blocks +WHERE hash = ? +LIMIT 1; + +-- name: SearchBlocks :many +SELECT + b.*, + COUNT(t.block_index) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (sqlc.arg(chain_id) = '' OR b.chain_id = sqlc.arg(chain_id)) + AND LENGTH(sqlc.arg(hash_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(b.hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (sqlc.arg(proposer_address) = '' OR LOWER(HEX(b.proposer_address)) = LOWER(sqlc.arg(proposer_address))) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT sqlc.arg(limit) +OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 0e625a197..c9c152482 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -1,14 +1,18 @@ -- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ? -); + ?, ?, ?, ?, ?, ?, ?, ? +) +ON CONFLICT(hash) DO UPDATE +SET block_height = excluded.block_height, + block_index = excluded.block_index, + type = excluded.type, + subtype = excluded.subtype, + raw_tx = excluded.raw_tx, + signature = excluded.signature, + signer = excluded.signer; --- name: GetTransaction :one -SELECT * FROM transactions -WHERE id = ? -LIMIT 1; -- name: GetTransactionByHash :one SELECT * FROM transactions @@ -18,22 +22,29 @@ LIMIT 1; -- name: CountTransactions :one SELECT COUNT(*) FROM transactions; --- name: GetTxReferenceByBlockHeightAndBlockIndex :one +-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ?; + +-- name: GetTransactionByHeightAndIndex :one SELECT * FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1; -- name: SearchTransactions :many -WITH results AS ( - SELECT * - FROM transactions - WHERE ( - (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) - AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) - ) -) SELECT *, COUNT(*) OVER() AS total_count -FROM results -ORDER BY id DESC +FROM transactions +WHERE + (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) + AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(tx_subtype) = '' OR LOWER(subtype) = LOWER(sqlc.arg(tx_subtype))) + AND (sqlc.arg(tx_signer) = '' OR LOWER(HEX(signer)) = LOWER(sqlc.arg(tx_signer))) + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) +ORDER BY block_height DESC, block_index DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 4b8c9fe21..31beb4465 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -5,12 +5,15 @@ import ( "database/sql" "errors" "fmt" + "strings" + "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/types" indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" + "google.golang.org/protobuf/proto" ) // ErrTransactionNotFound is returned if the transaction is not found. @@ -22,21 +25,26 @@ func (idx *Indexer) CountTotalTransactions() (uint64, error) { return uint64(count), err } -// GetTransaction fetches the txReference for the given tx height -func (idx *Indexer) GetTransaction(id uint64) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransaction(context.TODO(), int64(id)) +// CountTransactionsByHeight returns the number of transactions indexed for a given height +func (idx *Indexer) CountTransactionsByHeight(height int64) (int64, error) { + return idx.readOnlyQuery.CountTransactionsByHeight(context.TODO(), height) +} + +// GetTxMetadataByHash fetches the tx metadata for the given tx hash +func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.TransactionMetadata, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTransactionNotFound } - return nil, fmt.Errorf("tx with id %d not found: %v", id, err) + return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) } - return indexertypes.TransactionFromDB(&sqlTxRef), nil + return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil } -// GetTxReferenceByBlockHeightAndBlockIndex fetches the txReference for the given tx height and block tx index -func (idx *Indexer) GetTxReferenceByBlockHeightAndBlockIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTxReferenceByBlockHeightAndBlockIndex(context.TODO(), indexerdb.GetTxReferenceByBlockHeightAndBlockIndexParams{ +// GetTransactionByHeightAndIndex fetches the full tx for the given tx height and block tx index +func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHeightAndIndex(context.TODO(), indexerdb.GetTransactionByHeightAndIndexParams{ BlockHeight: blockHeight, BlockIndex: blockIndex, }) @@ -49,22 +57,11 @@ func (idx *Indexer) GetTxReferenceByBlockHeightAndBlockIndex(blockHeight, blockI return indexertypes.TransactionFromDB(&sqlTxRef), nil } -// GetTxHashReference fetches the txReference for the given tx hash -func (idx *Indexer) GetTxHashReference(hash types.HexBytes) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, ErrTransactionNotFound - } - return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) - } - return indexertypes.TransactionFromDB(&sqlTxRef), nil -} - // SearchTransactions returns the list of transactions indexed. -// height and txType are optional, if declared as zero-value will be ignored. +// blockHeight, hash, txType, txSubtype and txSigner are optional, if declared as zero-value will be ignored. +// hash matches substrings. // The first one returned is the newest, so they are in descending order. -func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType string) ([]*indexertypes.Transaction, uint64, error) { +func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txHash, txType, txSubtype, txSigner string) ([]*indexertypes.TransactionMetadata, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } @@ -74,21 +71,18 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx results, err := idx.readOnlyQuery.SearchTransactions(context.TODO(), indexerdb.SearchTransactionsParams{ Limit: int64(limit), Offset: int64(offset), + HashSubstr: txHash, BlockHeight: blockHeight, TxType: txType, + TxSubtype: txSubtype, + TxSigner: txSigner, }) if err != nil { return nil, 0, err } - list := []*indexertypes.Transaction{} + list := []*indexertypes.TransactionMetadata{} for _, row := range results { - list = append(list, &indexertypes.Transaction{ - Index: uint64(row.ID), - Hash: row.Hash, - BlockHeight: uint32(row.BlockHeight), - TxBlockIndex: int32(row.BlockIndex), - TxType: row.Type, - }) + list = append(list, indexertypes.TransactionMetadataFromDBRow(&row)) } if len(results) == 0 { return list, 0, nil @@ -99,13 +93,38 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { idx.blockMu.Lock() defer idx.blockMu.Unlock() + + idx.indexTx(tx, blockHeight, txIndex) +} + +func (idx *Indexer) indexTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { + rawtx, err := proto.Marshal(tx.Tx) + if err != nil { + log.Errorw(err, "indexer cannot marshal transaction") + return + } + + signer := []byte{} + if len(tx.Signature) > 0 { // not all txs are signed, for example zk ones + addr, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) + if err != nil { + log.Errorw(err, "indexer cannot recover signer from signature") + return + } + signer = addr.Bytes() + } + queries := idx.blockTxQueries() if _, err := queries.CreateTransaction(context.TODO(), indexerdb.CreateTransactionParams{ Hash: tx.TxID[:], BlockHeight: int64(blockHeight), BlockIndex: int64(txIndex), Type: tx.TxModelType, + Subtype: strings.ToLower(tx.TxSubtype()), + RawTx: rawtx, + Signature: nonNullBytes(tx.Signature), + Signer: nonNullBytes(signer), }); err != nil { - log.Errorw(err, "cannot index new transaction") + log.Errorw(err, "cannot index transaction") } } diff --git a/vochain/keykeeper/keykeeper.go b/vochain/keykeeper/keykeeper.go index 7f46f654d..4a4425cd1 100644 --- a/vochain/keykeeper/keykeeper.go +++ b/vochain/keykeeper/keykeeper.go @@ -268,9 +268,6 @@ func (*KeyKeeper) OnVote(_ *state.Vote, _ int32) {} // OnNewTx is not used by the KeyKeeper func (*KeyKeeper) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -// OnBeginBlock is not used by the KeyKeeper -func (*KeyKeeper) OnBeginBlock(_ state.BeginBlock) {} - // OnCensusUpdate is not used by the KeyKeeper func (*KeyKeeper) OnCensusUpdate(_, _ []byte, _ string, _ uint64) {} diff --git a/vochain/offchaindatahandler/offchaindatahandler.go b/vochain/offchaindatahandler/offchaindatahandler.go index d6e97bd9b..4bc3be6b4 100644 --- a/vochain/offchaindatahandler/offchaindatahandler.go +++ b/vochain/offchaindatahandler/offchaindatahandler.go @@ -166,7 +166,6 @@ func (d *OffChainDataHandler) OnSetAccount(_ []byte, account *state.Account) { func (*OffChainDataHandler) OnCancel(_ []byte, _ int32) {} func (*OffChainDataHandler) OnVote(_ *state.Vote, _ int32) {} func (*OffChainDataHandler) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*OffChainDataHandler) OnBeginBlock(state.BeginBlock) {} func (*OffChainDataHandler) OnProcessKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnRevealKeys(_ []byte, _ string, _ int32) {} func (*OffChainDataHandler) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} diff --git a/vochain/state/eventlistener.go b/vochain/state/eventlistener.go index 1c2d05281..7b599702b 100644 --- a/vochain/state/eventlistener.go +++ b/vochain/state/eventlistener.go @@ -1,8 +1,6 @@ package state import ( - "time" - "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" ) @@ -32,7 +30,6 @@ type EventListener interface { OnSpendTokens(addr []byte, txType models.TxType, cost uint64, reference string) OnCensusUpdate(pid, censusRoot []byte, censusURI string, censusSize uint64) Commit(height uint32) (err error) - OnBeginBlock(BeginBlock) Rollback() } @@ -46,15 +43,3 @@ func (v *State) AddEventListener(l EventListener) { func (v *State) CleanEventListeners() { v.eventListeners = nil } - -type BeginBlock struct { - Height int64 - Time time.Time - DataHash []byte -} - -func (v *State) OnBeginBlock(bb BeginBlock) { - for _, l := range v.eventListeners { - l.OnBeginBlock(bb) - } -} diff --git a/vochain/state/state_test.go b/vochain/state/state_test.go index e2b76c685..b36dc7722 100644 --- a/vochain/state/state_test.go +++ b/vochain/state/state_test.go @@ -182,7 +182,6 @@ type Listener struct { func (*Listener) OnVote(_ *Vote, _ int32) {} func (*Listener) OnNewTx(_ *vochaintx.Tx, _ uint32, _ int32) {} -func (*Listener) OnBeginBlock(BeginBlock) {} func (*Listener) OnProcess(_ *models.Process, _ int32) {} func (*Listener) OnProcessStatusChange(_ []byte, _ models.ProcessStatus, _ int32) {} func (*Listener) OnProcessDurationChange(_ []byte, _ uint32, _ int32) {} diff --git a/vochain/transaction/vochaintx/vochaintx.go b/vochain/transaction/vochaintx/vochaintx.go index f413e5e3b..66bc950fe 100644 --- a/vochain/transaction/vochaintx/vochaintx.go +++ b/vochain/transaction/vochaintx/vochaintx.go @@ -50,6 +50,32 @@ func (tx *Tx) Unmarshal(content []byte, chainID string) error { return nil } +// TxSubtype returns the content of the "txtype" field inside the tx.Tx. +// +// The function determines the type of the transaction using Protocol Buffers reflection. +// If the field doesn't exist, it returns the empty string "". +func (tx *Tx) TxSubtype() string { + txReflectDescriptor := tx.Tx.ProtoReflect().Descriptor().Oneofs().Get(0) + if txReflectDescriptor == nil { + return "" + } + whichOneTxModelType := tx.Tx.ProtoReflect().WhichOneof(txReflectDescriptor) + if whichOneTxModelType == nil { + return "" + } + // Get the value of the selected field in the oneof + fieldValue := tx.Tx.ProtoReflect().Get(whichOneTxModelType) + // Now, fieldValue is a protoreflect.Value, retrieve the txtype field + txtypeFieldDescriptor := fieldValue.Message().Descriptor().Fields().ByName("txtype") + if txtypeFieldDescriptor == nil { + return "" + } + // Get the integer value of txtype as protoreflect.EnumNumber + enumNumber := fieldValue.Message().Get(txtypeFieldDescriptor).Enum() + // Convert the EnumNumber to a string using the EnumType descriptor + return string(txtypeFieldDescriptor.Enum().Values().ByNumber(enumNumber).Name()) +} + // TxKey computes the checksum of the tx func TxKey(tx []byte) [32]byte { return comettypes.Tx(tx).Key()