From e9c571cb999f0b1bf9a558819001c6cca2641b68 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Thu, 12 Sep 2024 12:03:58 +0200 Subject: [PATCH] indexer: optimize performance of List methods * idx.TokenFeesList * idx.TokenTransfersList * idx.ProcessList * idx.TransactionList (renamed from idx.SearchTransactions) * idx.VoteList * idx.AccountList --- api/chain.go | 2 +- vochain/indexer/db/account.sql.go | 63 +++---- vochain/indexer/db/db.go | 40 +++- vochain/indexer/db/models.go | 16 ++ vochain/indexer/db/processes.sql.go | 197 +++++++++++++------- vochain/indexer/db/token_fees.sql.go | 69 ++++--- vochain/indexer/db/token_transfers.sql.go | 77 ++++---- vochain/indexer/db/transactions.sql.go | 54 ++++-- vochain/indexer/db/votes.sql.go | 94 ++++++---- vochain/indexer/indexer.go | 31 ++- vochain/indexer/indexer_test.go | 4 +- vochain/indexer/indexertypes/types.go | 11 -- vochain/indexer/process.go | 32 ++-- vochain/indexer/queries/account.sql | 31 +-- vochain/indexer/queries/processes.sql | 126 ++++++++----- vochain/indexer/queries/token_fees.sql | 25 ++- vochain/indexer/queries/token_transfers.sql | 32 ++-- vochain/indexer/queries/transactions.sql | 17 +- vochain/indexer/queries/votes.sql | 63 ++++--- vochain/indexer/transaction.go | 21 ++- vochain/indexer/vote.go | 12 +- 21 files changed, 625 insertions(+), 392 deletions(-) diff --git a/api/chain.go b/api/chain.go index 931b4bf21..4f261d6da 100644 --- a/api/chain.go +++ b/api/chain.go @@ -862,7 +862,7 @@ func (a *API) chainTxListByHeightAndPageHandler(_ *apirest.APIdata, ctx *httprou // // Errors returned are always of type APIerror. func (a *API) transactionList(params *TransactionParams) (*TransactionsList, error) { - txs, total, err := a.indexer.SearchTransactions( + txs, total, err := a.indexer.TransactionList( params.Limit, params.Page*params.Limit, params.Height, diff --git a/vochain/indexer/db/account.sql.go b/vochain/indexer/db/account.sql.go index 75ceffc85..5d8df4c9e 100644 --- a/vochain/indexer/db/account.sql.go +++ b/vochain/indexer/db/account.sql.go @@ -13,11 +13,20 @@ import ( ) const countAccounts = `-- name: CountAccounts :one -SELECT COUNT(*) FROM accounts +SELECT COUNT(*) +FROM accounts +WHERE ( + ( + ?1 = '' + OR (LENGTH(?1) = 40 AND LOWER(HEX(account)) = LOWER(?1)) + OR (LENGTH(?1) < 40 AND INSTR(LOWER(HEX(account)), LOWER(?1)) > 0) + -- TODO: consider keeping an account_hex column for faster searches + ) +) ` -func (q *Queries) CountAccounts(ctx context.Context) (int64, error) { - row := q.queryRow(ctx, q.countAccountsStmt, countAccounts) +func (q *Queries) CountAccounts(ctx context.Context, accountIDSubstr interface{}) (int64, error) { + row := q.queryRow(ctx, q.countAccountsStmt, countAccounts, accountIDSubstr) var count int64 err := row.Scan(&count) return count, err @@ -40,53 +49,37 @@ func (q *Queries) CreateAccount(ctx context.Context, arg CreateAccountParams) (s } const searchAccounts = `-- name: SearchAccounts :many -WITH results AS ( - SELECT account, balance, nonce - FROM accounts - WHERE ( - ( - ?3 = '' - OR (LENGTH(?3) = 40 AND LOWER(HEX(account)) = LOWER(?3)) - OR (LENGTH(?3) < 40 AND INSTR(LOWER(HEX(account)), LOWER(?3)) > 0) - -- TODO: consider keeping an account_hex column for faster searches - ) +SELECT account, balance, nonce +FROM accounts +WHERE ( + ( + ?1 = '' + OR (LENGTH(?1) = 40 AND LOWER(HEX(account)) = LOWER(?1)) + OR (LENGTH(?1) < 40 AND INSTR(LOWER(HEX(account)), LOWER(?1)) > 0) + -- TODO: consider keeping an account_hex column for faster searches ) ) -SELECT account, balance, nonce, COUNT(*) OVER() AS total_count -FROM results ORDER BY balance DESC -LIMIT ?2 -OFFSET ?1 +LIMIT ?3 +OFFSET ?2 ` type SearchAccountsParams struct { + AccountIDSubstr interface{} Offset int64 Limit int64 - AccountIDSubstr interface{} -} - -type SearchAccountsRow struct { - Account []byte - Balance int64 - Nonce int64 - TotalCount int64 } -func (q *Queries) SearchAccounts(ctx context.Context, arg SearchAccountsParams) ([]SearchAccountsRow, error) { - rows, err := q.query(ctx, q.searchAccountsStmt, searchAccounts, arg.Offset, arg.Limit, arg.AccountIDSubstr) +func (q *Queries) SearchAccounts(ctx context.Context, arg SearchAccountsParams) ([]Account, error) { + rows, err := q.query(ctx, q.searchAccountsStmt, searchAccounts, arg.AccountIDSubstr, arg.Offset, arg.Limit) if err != nil { return nil, err } defer rows.Close() - var items []SearchAccountsRow + var items []Account for rows.Next() { - var i SearchAccountsRow - if err := rows.Scan( - &i.Account, - &i.Balance, - &i.Nonce, - &i.TotalCount, - ); err != nil { + var i Account + if err := rows.Scan(&i.Account, &i.Balance, &i.Nonce); err != nil { return nil, err } items = append(items, i) diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index bcbb12f84..59fdb0e05 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -33,6 +33,15 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.countBlocksStmt, err = db.PrepareContext(ctx, countBlocks); err != nil { return nil, fmt.Errorf("error preparing query CountBlocks: %w", err) } + if q.countProcessesStmt, err = db.PrepareContext(ctx, countProcesses); err != nil { + return nil, fmt.Errorf("error preparing query CountProcesses: %w", err) + } + if q.countTokenFeesStmt, err = db.PrepareContext(ctx, countTokenFees); err != nil { + return nil, fmt.Errorf("error preparing query CountTokenFees: %w", err) + } + if q.countTokenTransfersStmt, err = db.PrepareContext(ctx, countTokenTransfers); err != nil { + return nil, fmt.Errorf("error preparing query CountTokenTransfers: %w", err) + } if q.countTokenTransfersByAccountStmt, err = db.PrepareContext(ctx, countTokenTransfersByAccount); err != nil { return nil, fmt.Errorf("error preparing query CountTokenTransfersByAccount: %w", err) } @@ -78,9 +87,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getProcessStmt, err = db.PrepareContext(ctx, getProcess); err != nil { return nil, fmt.Errorf("error preparing query GetProcess: %w", err) } - if q.getProcessCountStmt, err = db.PrepareContext(ctx, getProcessCount); err != nil { - return nil, fmt.Errorf("error preparing query GetProcessCount: %w", err) - } if q.getProcessIDsByFinalResultsStmt, err = db.PrepareContext(ctx, getProcessIDsByFinalResults); err != nil { return nil, fmt.Errorf("error preparing query GetProcessIDsByFinalResults: %w", err) } @@ -164,6 +170,21 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing countBlocksStmt: %w", cerr) } } + if q.countProcessesStmt != nil { + if cerr := q.countProcessesStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countProcessesStmt: %w", cerr) + } + } + if q.countTokenFeesStmt != nil { + if cerr := q.countTokenFeesStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countTokenFeesStmt: %w", cerr) + } + } + if q.countTokenTransfersStmt != nil { + if cerr := q.countTokenTransfersStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countTokenTransfersStmt: %w", cerr) + } + } if q.countTokenTransfersByAccountStmt != nil { if cerr := q.countTokenTransfersByAccountStmt.Close(); cerr != nil { err = fmt.Errorf("error closing countTokenTransfersByAccountStmt: %w", cerr) @@ -239,11 +260,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getProcessStmt: %w", cerr) } } - if q.getProcessCountStmt != nil { - if cerr := q.getProcessCountStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getProcessCountStmt: %w", cerr) - } - } if q.getProcessIDsByFinalResultsStmt != nil { if cerr := q.getProcessIDsByFinalResultsStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getProcessIDsByFinalResultsStmt: %w", cerr) @@ -391,6 +407,9 @@ type Queries struct { computeProcessVoteCountStmt *sql.Stmt countAccountsStmt *sql.Stmt countBlocksStmt *sql.Stmt + countProcessesStmt *sql.Stmt + countTokenFeesStmt *sql.Stmt + countTokenTransfersStmt *sql.Stmt countTokenTransfersByAccountStmt *sql.Stmt countTransactionsStmt *sql.Stmt countTransactionsByHeightStmt *sql.Stmt @@ -406,7 +425,6 @@ type Queries struct { getBlockByHeightStmt *sql.Stmt getEntityCountStmt *sql.Stmt getProcessStmt *sql.Stmt - getProcessCountStmt *sql.Stmt getProcessIDsByFinalResultsStmt *sql.Stmt getProcessStatusStmt *sql.Stmt getTokenTransferStmt *sql.Stmt @@ -437,6 +455,9 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, countAccountsStmt: q.countAccountsStmt, countBlocksStmt: q.countBlocksStmt, + countProcessesStmt: q.countProcessesStmt, + countTokenFeesStmt: q.countTokenFeesStmt, + countTokenTransfersStmt: q.countTokenTransfersStmt, countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, countTransactionsStmt: q.countTransactionsStmt, countTransactionsByHeightStmt: q.countTransactionsByHeightStmt, @@ -452,7 +473,6 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getBlockByHeightStmt: q.getBlockByHeightStmt, getEntityCountStmt: q.getEntityCountStmt, getProcessStmt: q.getProcessStmt, - getProcessCountStmt: q.getProcessCountStmt, getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, getProcessStatusStmt: q.getProcessStatusStmt, getTokenTransferStmt: q.getTokenTransferStmt, diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 2f25a2817..a02d3eedd 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -10,6 +10,12 @@ import ( "go.vocdoni.io/dvote/types" ) +type Account struct { + Account types.AccountID + Balance int64 + Nonce int64 +} + type Block struct { Height int64 Time time.Time @@ -50,6 +56,16 @@ type Process struct { ManuallyEnded bool } +type TokenFee struct { + ID int64 + BlockHeight int64 + FromAccount []byte + Reference string + Cost int64 + TxType string + SpendTime time.Time +} + type TokenTransfer struct { TxHash types.Hash BlockHeight int64 diff --git a/vochain/indexer/db/processes.sql.go b/vochain/indexer/db/processes.sql.go index 66ed2861b..250df12e6 100644 --- a/vochain/indexer/db/processes.sql.go +++ b/vochain/indexer/db/processes.sql.go @@ -23,6 +23,84 @@ func (q *Queries) ComputeProcessVoteCount(ctx context.Context, id types.ProcessI return q.exec(ctx, q.computeProcessVoteCountStmt, computeProcessVoteCount, id) } +const countProcesses = `-- name: CountProcesses :one +SELECT COUNT(*) +FROM processes +WHERE ( + LENGTH(?1) <= 40 -- if passed arg is longer, then just abort the query + AND ( + ?1 = '' + OR (LENGTH(?1) = 40 AND LOWER(HEX(entity_id)) = LOWER(?1)) + OR (LENGTH(?1) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(?1)) > 0) + -- TODO: consider keeping an entity_id_hex column for faster searches + ) + AND (?2 = 0 OR namespace = ?2) + AND (?3 = 0 OR status = ?3) + AND (?4 = 0 OR source_network_id = ?4) + AND LENGTH(?5) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?5 = '' + OR (LENGTH(?5) = 64 AND LOWER(HEX(id)) = LOWER(?5)) + OR (LENGTH(?5) < 64 AND INSTR(LOWER(HEX(id)), LOWER(?5)) > 0) + -- TODO: consider keeping an id_hex column for faster searches + ) + AND ( + ?6 = -1 + OR (?6 = 1 AND have_results = TRUE) + OR (?6 = 0 AND have_results = FALSE) + ) + AND ( + ?7 = -1 + OR (?7 = 1 AND final_results = TRUE) + OR (?7 = 0 AND final_results = FALSE) + ) + AND ( + ?8 = -1 + OR (?8 = 1 AND manually_ended = TRUE) + OR (?8 = 0 AND manually_ended = FALSE) + ) + AND (?9 IS NULL OR start_date >= ?9) + AND (?10 IS NULL OR start_date <= ?10) + AND (?11 IS NULL OR end_date >= ?11) + AND (?12 IS NULL OR end_date <= ?12) +) +` + +type CountProcessesParams struct { + EntityIDSubstr interface{} + Namespace interface{} + Status interface{} + SourceNetworkID interface{} + IDSubstr interface{} + HaveResults interface{} + FinalResults interface{} + ManuallyEnded interface{} + StartDateAfter interface{} + StartDateBefore interface{} + EndDateAfter interface{} + EndDateBefore interface{} +} + +func (q *Queries) CountProcesses(ctx context.Context, arg CountProcessesParams) (int64, error) { + row := q.queryRow(ctx, q.countProcessesStmt, countProcesses, + arg.EntityIDSubstr, + arg.Namespace, + arg.Status, + arg.SourceNetworkID, + arg.IDSubstr, + arg.HaveResults, + arg.FinalResults, + arg.ManuallyEnded, + arg.StartDateAfter, + arg.StartDateBefore, + arg.EndDateAfter, + arg.EndDateBefore, + ) + var count int64 + err := row.Scan(&count) + return count, err +} + const createProcess = `-- name: CreateProcess :execresult INSERT INTO processes ( id, entity_id, start_date, end_date, manually_ended, @@ -164,17 +242,6 @@ func (q *Queries) GetProcess(ctx context.Context, id types.ProcessID) (Process, return i, err } -const getProcessCount = `-- name: GetProcessCount :one -SELECT COUNT(*) FROM processes -` - -func (q *Queries) GetProcessCount(ctx context.Context) (int64, error) { - row := q.queryRow(ctx, q.getProcessCountStmt, getProcessCount) - var count int64 - err := row.Scan(&count) - return count, err -} - const getProcessIDsByFinalResults = `-- name: GetProcessIDsByFinalResults :many SELECT id FROM processes WHERE final_results = ? @@ -268,59 +335,52 @@ func (q *Queries) SearchEntities(ctx context.Context, arg SearchEntitiesParams) } const searchProcesses = `-- name: SearchProcesses :many -WITH results AS ( - SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, - COUNT(*) OVER() AS total_count - FROM processes - WHERE ( - LENGTH(?3) <= 40 -- if passed arg is longer, then just abort the query - AND ( - ?3 = '' - OR (LENGTH(?3) = 40 AND LOWER(HEX(entity_id)) = LOWER(?3)) - OR (LENGTH(?3) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(?3)) > 0) - -- TODO: consider keeping an entity_id_hex column for faster searches - ) - AND (?4 = 0 OR namespace = ?4) - AND (?5 = 0 OR status = ?5) - AND (?6 = 0 OR source_network_id = ?6) - AND LENGTH(?7) <= 64 -- if passed arg is longer, then just abort the query - AND ( - ?7 = '' - OR (LENGTH(?7) = 64 AND LOWER(HEX(id)) = LOWER(?7)) - OR (LENGTH(?7) < 64 AND INSTR(LOWER(HEX(id)), LOWER(?7)) > 0) - -- TODO: consider keeping an id_hex column for faster searches - ) - AND ( - ?8 = -1 - OR (?8 = 1 AND have_results = TRUE) - OR (?8 = 0 AND have_results = FALSE) - ) - AND ( - ?9 = -1 - OR (?9 = 1 AND final_results = TRUE) - OR (?9 = 0 AND final_results = FALSE) - ) - AND ( - ?10 = -1 - OR (?10 = 1 AND manually_ended = TRUE) - OR (?10 = 0 AND manually_ended = FALSE) - ) - AND (?11 IS NULL OR start_date >= ?11) - AND (?12 IS NULL OR start_date <= ?12) - AND (?13 IS NULL OR end_date >= ?13) - AND (?14 IS NULL OR end_date <= ?14) +SELECT id +FROM processes +WHERE ( + LENGTH(?1) <= 40 -- if passed arg is longer, then just abort the query + AND ( + ?1 = '' + OR (LENGTH(?1) = 40 AND LOWER(HEX(entity_id)) = LOWER(?1)) + OR (LENGTH(?1) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(?1)) > 0) + -- TODO: consider keeping an entity_id_hex column for faster searches + ) + AND (?2 = 0 OR namespace = ?2) + AND (?3 = 0 OR status = ?3) + AND (?4 = 0 OR source_network_id = ?4) + AND LENGTH(?5) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?5 = '' + OR (LENGTH(?5) = 64 AND LOWER(HEX(id)) = LOWER(?5)) + OR (LENGTH(?5) < 64 AND INSTR(LOWER(HEX(id)), LOWER(?5)) > 0) + -- TODO: consider keeping an id_hex column for faster searches ) + AND ( + ?6 = -1 + OR (?6 = 1 AND have_results = TRUE) + OR (?6 = 0 AND have_results = FALSE) + ) + AND ( + ?7 = -1 + OR (?7 = 1 AND final_results = TRUE) + OR (?7 = 0 AND final_results = FALSE) + ) + AND ( + ?8 = -1 + OR (?8 = 1 AND manually_ended = TRUE) + OR (?8 = 0 AND manually_ended = FALSE) + ) + AND (?9 IS NULL OR start_date >= ?9) + AND (?10 IS NULL OR start_date <= ?10) + AND (?11 IS NULL OR end_date >= ?11) + AND (?12 IS NULL OR end_date <= ?12) ) -SELECT id, total_count -FROM results ORDER BY creation_time DESC, id ASC -LIMIT ?2 -OFFSET ?1 +LIMIT ?14 +OFFSET ?13 ` type SearchProcessesParams struct { - Offset int64 - Limit int64 EntityIDSubstr interface{} Namespace interface{} Status interface{} @@ -333,17 +393,12 @@ type SearchProcessesParams struct { StartDateBefore interface{} EndDateAfter interface{} EndDateBefore interface{} + Offset int64 + Limit int64 } -type SearchProcessesRow struct { - ID []byte - TotalCount int64 -} - -func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams) ([]SearchProcessesRow, error) { +func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams) ([]types.ProcessID, error) { rows, err := q.query(ctx, q.searchProcessesStmt, searchProcesses, - arg.Offset, - arg.Limit, arg.EntityIDSubstr, arg.Namespace, arg.Status, @@ -356,18 +411,20 @@ func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams arg.StartDateBefore, arg.EndDateAfter, arg.EndDateBefore, + arg.Offset, + arg.Limit, ) if err != nil { return nil, err } defer rows.Close() - var items []SearchProcessesRow + var items []types.ProcessID for rows.Next() { - var i SearchProcessesRow - if err := rows.Scan(&i.ID, &i.TotalCount); err != nil { + var id types.ProcessID + if err := rows.Scan(&id); err != nil { return nil, err } - items = append(items, i) + items = append(items, id) } if err := rows.Close(); err != nil { return nil, err diff --git a/vochain/indexer/db/token_fees.sql.go b/vochain/indexer/db/token_fees.sql.go index ef44efbe4..af0ad84ec 100644 --- a/vochain/indexer/db/token_fees.sql.go +++ b/vochain/indexer/db/token_fees.sql.go @@ -11,6 +11,29 @@ import ( "time" ) +const countTokenFees = `-- name: CountTokenFees :one +SELECT COUNT(*) +FROM token_fees +WHERE ( + (?1 = '' OR LOWER(HEX(from_account)) = LOWER(?1)) + AND (?2 = '' OR LOWER(tx_type) = LOWER(?2)) + AND (?3 = '' OR LOWER(reference) = LOWER(?3)) +) +` + +type CountTokenFeesParams struct { + FromAccount interface{} + TxType interface{} + Reference interface{} +} + +func (q *Queries) CountTokenFees(ctx context.Context, arg CountTokenFeesParams) (int64, error) { + row := q.queryRow(ctx, q.countTokenFeesStmt, countTokenFees, arg.FromAccount, arg.TxType, arg.Reference) + var count int64 + err := row.Scan(&count) + return count, err +} + const createTokenFee = `-- name: CreateTokenFee :execresult INSERT INTO token_fees ( from_account, block_height, reference, @@ -42,56 +65,41 @@ func (q *Queries) CreateTokenFee(ctx context.Context, arg CreateTokenFeeParams) } const searchTokenFees = `-- name: SearchTokenFees :many -WITH results AS ( - SELECT id, block_height, from_account, reference, cost, tx_type, spend_time - FROM token_fees - WHERE ( - (?3 = '' OR LOWER(HEX(from_account)) = LOWER(?3)) - AND (?4 = '' OR LOWER(tx_type) = LOWER(?4)) - AND (?5 = '' OR LOWER(reference) = LOWER(?5)) - ) +SELECT id, block_height, from_account, reference, cost, tx_type, spend_time +FROM token_fees +WHERE ( + (?1 = '' OR LOWER(HEX(from_account)) = LOWER(?1)) + AND (?2 = '' OR LOWER(tx_type) = LOWER(?2)) + AND (?3 = '' OR LOWER(reference) = LOWER(?3)) ) -SELECT id, block_height, from_account, reference, cost, tx_type, spend_time, COUNT(*) OVER() AS total_count -FROM results ORDER BY spend_time DESC -LIMIT ?2 -OFFSET ?1 +LIMIT ?5 +OFFSET ?4 ` type SearchTokenFeesParams struct { - Offset int64 - Limit int64 FromAccount interface{} TxType interface{} Reference interface{} + Offset int64 + Limit int64 } -type SearchTokenFeesRow struct { - ID int64 - BlockHeight int64 - FromAccount []byte - Reference string - Cost int64 - TxType string - SpendTime time.Time - TotalCount int64 -} - -func (q *Queries) SearchTokenFees(ctx context.Context, arg SearchTokenFeesParams) ([]SearchTokenFeesRow, error) { +func (q *Queries) SearchTokenFees(ctx context.Context, arg SearchTokenFeesParams) ([]TokenFee, error) { rows, err := q.query(ctx, q.searchTokenFeesStmt, searchTokenFees, - arg.Offset, - arg.Limit, arg.FromAccount, arg.TxType, arg.Reference, + arg.Offset, + arg.Limit, ) if err != nil { return nil, err } defer rows.Close() - var items []SearchTokenFeesRow + var items []TokenFee for rows.Next() { - var i SearchTokenFeesRow + var i TokenFee if err := rows.Scan( &i.ID, &i.BlockHeight, @@ -100,7 +108,6 @@ func (q *Queries) SearchTokenFees(ctx context.Context, arg SearchTokenFeesParams &i.Cost, &i.TxType, &i.SpendTime, - &i.TotalCount, ); err != nil { return nil, err } diff --git a/vochain/indexer/db/token_transfers.sql.go b/vochain/indexer/db/token_transfers.sql.go index 2d8a1d129..084a7bfcf 100644 --- a/vochain/indexer/db/token_transfers.sql.go +++ b/vochain/indexer/db/token_transfers.sql.go @@ -13,6 +13,32 @@ import ( "go.vocdoni.io/dvote/types" ) +const countTokenTransfers = `-- name: CountTokenTransfers :one +SELECT COUNT(*) +FROM token_transfers +WHERE ( + (?1 = '' + OR LOWER(HEX(from_account)) = LOWER(?1) + OR LOWER(HEX(to_account)) = LOWER(?1) + ) + AND (?2 = '' OR LOWER(HEX(from_account)) = LOWER(?2)) + AND (?3 = '' OR LOWER(HEX(to_account)) = LOWER(?3)) +) +` + +type CountTokenTransfersParams struct { + FromOrToAccount interface{} + FromAccount interface{} + ToAccount interface{} +} + +func (q *Queries) CountTokenTransfers(ctx context.Context, arg CountTokenTransfersParams) (int64, error) { + row := q.queryRow(ctx, q.countTokenTransfersStmt, countTokenTransfers, arg.FromOrToAccount, arg.FromAccount, arg.ToAccount) + var count int64 + err := row.Scan(&count) + return count, err +} + const countTokenTransfersByAccount = `-- name: CountTokenTransfersByAccount :one SELECT COUNT(*) FROM token_transfers WHERE to_account = ?1 OR @@ -77,58 +103,44 @@ func (q *Queries) GetTokenTransfer(ctx context.Context, txHash types.Hash) (Toke } const searchTokenTransfers = `-- name: SearchTokenTransfers :many -WITH results AS ( - SELECT tx_hash, block_height, from_account, to_account, amount, transfer_time - FROM token_transfers - WHERE ( - (?3 = '' OR ( - LOWER(HEX(from_account)) = LOWER(?3) - OR LOWER(HEX(to_account)) = LOWER(?3) - )) - AND (?4 = '' OR LOWER(HEX(from_account)) = LOWER(?4)) - AND (?5 = '' OR LOWER(HEX(to_account)) = LOWER(?5)) - ) +SELECT tx_hash, block_height, from_account, to_account, amount, transfer_time +FROM token_transfers +WHERE ( + (?1 = '' + OR LOWER(HEX(from_account)) = LOWER(?1) + OR LOWER(HEX(to_account)) = LOWER(?1) + ) + AND (?2 = '' OR LOWER(HEX(from_account)) = LOWER(?2)) + AND (?3 = '' OR LOWER(HEX(to_account)) = LOWER(?3)) ) -SELECT tx_hash, block_height, from_account, to_account, amount, transfer_time, COUNT(*) OVER() AS total_count -FROM results ORDER BY transfer_time DESC -LIMIT ?2 -OFFSET ?1 +LIMIT ?5 +OFFSET ?4 ` type SearchTokenTransfersParams struct { - Offset int64 - Limit int64 FromOrToAccount interface{} FromAccount interface{} ToAccount interface{} + Offset int64 + Limit int64 } -type SearchTokenTransfersRow struct { - TxHash []byte - BlockHeight int64 - FromAccount []byte - ToAccount []byte - Amount int64 - TransferTime time.Time - TotalCount int64 -} - -func (q *Queries) SearchTokenTransfers(ctx context.Context, arg SearchTokenTransfersParams) ([]SearchTokenTransfersRow, error) { +func (q *Queries) SearchTokenTransfers(ctx context.Context, arg SearchTokenTransfersParams) ([]TokenTransfer, error) { rows, err := q.query(ctx, q.searchTokenTransfersStmt, searchTokenTransfers, - arg.Offset, - arg.Limit, arg.FromOrToAccount, arg.FromAccount, arg.ToAccount, + arg.Offset, + arg.Limit, ) if err != nil { return nil, err } defer rows.Close() - var items []SearchTokenTransfersRow + var items []TokenTransfer for rows.Next() { - var i SearchTokenTransfersRow + var i TokenTransfer if err := rows.Scan( &i.TxHash, &i.BlockHeight, @@ -136,7 +148,6 @@ func (q *Queries) SearchTokenTransfers(ctx context.Context, arg SearchTokenTrans &i.ToAccount, &i.Amount, &i.TransferTime, - &i.TotalCount, ); err != nil { return nil, err } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index 909053e1c..321c74104 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -13,11 +13,38 @@ import ( ) const countTransactions = `-- name: CountTransactions :one -SELECT COUNT(*) FROM transactions +SELECT COUNT(*) +FROM transactions +WHERE ( + (?1 = 0 OR block_height = ?1) + AND (?2 = '' OR LOWER(type) = LOWER(?2)) + AND (?3 = '' OR LOWER(subtype) = LOWER(?3)) + AND (?4 = '' OR LOWER(HEX(signer)) = LOWER(?4)) + AND ( + ?5 = '' + OR (LENGTH(?5) = 64 AND LOWER(HEX(hash)) = LOWER(?5)) + OR (LENGTH(?5) < 64 AND INSTR(LOWER(HEX(hash)), LOWER(?5)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) +) ` -func (q *Queries) CountTransactions(ctx context.Context) (int64, error) { - row := q.queryRow(ctx, q.countTransactionsStmt, countTransactions) +type CountTransactionsParams struct { + BlockHeight interface{} + TxType interface{} + TxSubtype interface{} + TxSigner interface{} + HashSubstr interface{} +} + +func (q *Queries) CountTransactions(ctx context.Context, arg CountTransactionsParams) (int64, error) { + row := q.queryRow(ctx, q.countTransactionsStmt, countTransactions, + arg.BlockHeight, + arg.TxType, + arg.TxSubtype, + arg.TxSigner, + arg.HashSubstr, + ) var count int64 err := row.Scan(&count) return count, err @@ -125,7 +152,7 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra } const searchTransactions = `-- name: SearchTransactions :many -SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer, COUNT(*) OVER() AS total_count +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE (?1 = 0 OR block_height = ?1) @@ -153,19 +180,7 @@ type SearchTransactionsParams struct { Limit int64 } -type SearchTransactionsRow struct { - Hash types.Hash - BlockHeight int64 - BlockIndex int64 - Type string - Subtype string - RawTx []byte - Signature []byte - Signer []byte - TotalCount int64 -} - -func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactionsParams) ([]SearchTransactionsRow, error) { +func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactionsParams) ([]Transaction, error) { rows, err := q.query(ctx, q.searchTransactionsStmt, searchTransactions, arg.BlockHeight, arg.TxType, @@ -179,9 +194,9 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions return nil, err } defer rows.Close() - var items []SearchTransactionsRow + var items []Transaction for rows.Next() { - var i SearchTransactionsRow + var i Transaction if err := rows.Scan( &i.Hash, &i.BlockHeight, @@ -191,7 +206,6 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions &i.RawTx, &i.Signature, &i.Signer, - &i.TotalCount, ); err != nil { return nil, err } diff --git a/vochain/indexer/db/votes.sql.go b/vochain/indexer/db/votes.sql.go index 48392a768..a85ad7589 100644 --- a/vochain/indexer/db/votes.sql.go +++ b/vochain/indexer/db/votes.sql.go @@ -14,11 +14,33 @@ import ( ) const countVotes = `-- name: CountVotes :one -SELECT COUNT(*) FROM votes +SELECT COUNT(*) +FROM votes +WHERE ( + LENGTH(?1) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?1 = '' + OR (LENGTH(?1) = 64 AND LOWER(HEX(process_id)) = LOWER(?1)) + OR (LENGTH(?1) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(?1)) > 0) + -- TODO: consider keeping an process_id_hex column for faster searches + ) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(nullifier)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(?2)) > 0) + -- TODO: consider keeping an nullifier_hex column for faster searches + ) +) ` -func (q *Queries) CountVotes(ctx context.Context) (int64, error) { - row := q.queryRow(ctx, q.countVotesStmt, countVotes) +type CountVotesParams struct { + ProcessIDSubstr interface{} + NullifierSubstr interface{} +} + +func (q *Queries) CountVotes(ctx context.Context, arg CountVotesParams) (int64, error) { + row := q.queryRow(ctx, q.countVotesStmt, countVotes, arg.ProcessIDSubstr, arg.NullifierSubstr) var count int64 err := row.Scan(&count) return count, err @@ -107,63 +129,58 @@ func (q *Queries) GetVote(ctx context.Context, nullifier types.Nullifier) (GetVo } const searchVotes = `-- name: SearchVotes :many -WITH results AS ( - SELECT v.nullifier, v.process_id, v.block_height, v.block_index, v.weight, v.voter_id, v.overwrite_count, v.encryption_key_indexes, v.package, t.hash - FROM votes AS v - LEFT JOIN transactions AS t - ON v.block_height = t.block_height - AND v.block_index = t.block_index - WHERE ( - LENGTH(?3) <= 64 -- if passed arg is longer, then just abort the query - AND ( - ?3 = '' - OR (LENGTH(?3) = 64 AND LOWER(HEX(process_id)) = LOWER(?3)) - OR (LENGTH(?3) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(?3)) > 0) - -- TODO: consider keeping an process_id_hex column for faster searches - ) - AND LENGTH(?4) <= 64 -- if passed arg is longer, then just abort the query - AND ( - ?4 = '' - OR (LENGTH(?4) = 64 AND LOWER(HEX(nullifier)) = LOWER(?4)) - OR (LENGTH(?4) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(?4)) > 0) - -- TODO: consider keeping an nullifier_hex column for faster searches - ) +SELECT v.nullifier, v.process_id, v.block_height, v.block_index, v.weight, v.voter_id, v.overwrite_count, v.encryption_key_indexes, v.package, t.hash +FROM votes AS v +LEFT JOIN transactions AS t + ON v.block_height = t.block_height + AND v.block_index = t.block_index +WHERE ( + LENGTH(?1) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?1 = '' + OR (LENGTH(?1) = 64 AND LOWER(HEX(process_id)) = LOWER(?1)) + OR (LENGTH(?1) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(?1)) > 0) + -- TODO: consider keeping an process_id_hex column for faster searches + ) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(nullifier)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(?2)) > 0) + -- TODO: consider keeping an nullifier_hex column for faster searches ) ) -SELECT nullifier, process_id, block_height, block_index, weight, voter_id, overwrite_count, encryption_key_indexes, package, hash, COUNT(*) OVER() AS total_count -FROM results -ORDER BY block_height DESC, nullifier ASC -LIMIT ?2 -OFFSET ?1 +ORDER BY v.block_height DESC, v.nullifier ASC +LIMIT ?4 +OFFSET ?3 ` type SearchVotesParams struct { - Offset int64 - Limit int64 ProcessIDSubstr interface{} NullifierSubstr interface{} + Offset int64 + Limit int64 } type SearchVotesRow struct { - Nullifier []byte - ProcessID []byte + Nullifier types.Nullifier + ProcessID types.ProcessID BlockHeight int64 BlockIndex int64 Weight string - VoterID []byte + VoterID state.VoterID OverwriteCount int64 EncryptionKeyIndexes string Package string - Hash []byte - TotalCount int64 + Hash types.Hash } func (q *Queries) SearchVotes(ctx context.Context, arg SearchVotesParams) ([]SearchVotesRow, error) { rows, err := q.query(ctx, q.searchVotesStmt, searchVotes, - arg.Offset, - arg.Limit, arg.ProcessIDSubstr, arg.NullifierSubstr, + arg.Offset, + arg.Limit, ) if err != nil { return nil, err @@ -183,7 +200,6 @@ func (q *Queries) SearchVotes(ctx context.Context, arg SearchVotesParams) ([]Sea &i.EncryptionKeyIndexes, &i.Package, &i.Hash, - &i.TotalCount, ); err != nil { return nil, err } diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 37fea31a2..34ebd16d3 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -887,10 +887,15 @@ func (idx *Indexer) TokenFeesList(limit, offset int, txType, reference, fromAcco Timestamp: row.SpendTime, }) } - if len(results) == 0 { - return list, 0, nil + count, err := idx.readOnlyQuery.CountTokenFees(context.TODO(), indexerdb.CountTokenFeesParams{ + TxType: txType, + Reference: reference, + FromAccount: fromAccount, + }) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, uint64(count), nil } // TokenTransfersList returns all the token transfers, made to and/or from a given account @@ -925,10 +930,15 @@ func (idx *Indexer) TokenTransfersList(limit, offset int, fromOrToAccount, fromA Timestamp: row.TransferTime, }) } - if len(results) == 0 { - return list, 0, nil + count, err := idx.readOnlyQuery.CountTokenTransfers(context.TODO(), indexerdb.CountTokenTransfersParams{ + FromOrToAccount: fromOrToAccount, + FromAccount: fromAccount, + ToAccount: toAccount, + }) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, uint64(count), nil } // CountTokenTransfersByAccount returns the count all the token transfers made from a given account @@ -939,7 +949,7 @@ func (idx *Indexer) CountTokenTransfersByAccount(acc []byte) (uint64, error) { // CountTotalAccounts returns the total number of accounts indexed. func (idx *Indexer) CountTotalAccounts() (uint64, error) { - count, err := idx.readOnlyQuery.CountAccounts(context.TODO()) + count, err := idx.readOnlyQuery.CountAccounts(context.TODO(), "") return uint64(count), err } @@ -968,10 +978,11 @@ func (idx *Indexer) AccountList(limit, offset int, accountID string) ([]*indexer Nonce: uint32(row.Nonce), }) } - if len(results) == 0 { - return list, 0, nil + count, err := idx.readOnlyQuery.CountAccounts(context.TODO(), accountID) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, uint64(count), nil } // AccountExists returns whether the passed accountID exists in the db. diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index f3dc00ed3..73bf70440 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1420,7 +1420,7 @@ func TestTxIndexer(t *testing.T) { } } - txs, _, err := idx.SearchTransactions(15, 0, 0, "", "", "", "") + txs, _, err := idx.TransactionList(15, 0, 0, "", "", "", "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { // BlockIndex and TxBlockIndex start at 0, so subtract 1. @@ -1429,7 +1429,7 @@ func TestTxIndexer(t *testing.T) { qt.Assert(t, tx.TxType, qt.Equals, "setAccount") } - txs, _, err = idx.SearchTransactions(1, 5, 0, "", "", "", "") + txs, _, err = idx.TransactionList(1, 5, 0, "", "", "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) } diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index ad5b29748..5133bedd8 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -197,17 +197,6 @@ func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata } } -func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { - return &TransactionMetadata{ - Hash: dbtx.Hash, - BlockHeight: uint32(dbtx.BlockHeight), - TxBlockIndex: int32(dbtx.BlockIndex), - TxType: dbtx.Type, - TxSubtype: dbtx.Subtype, - Signer: dbtx.Signer, - } -} - // Transaction holds a single transaction type Transaction struct { *TransactionMetadata diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go index 3a33fee32..3013f3948 100644 --- a/vochain/indexer/process.go +++ b/vochain/indexer/process.go @@ -63,14 +63,14 @@ func (idx *Indexer) ProcessList(limit, offset int, entityID string, processID st if _, ok := models.SourceNetworkId_name[srcNetworkID]; !ok { return nil, 0, fmt.Errorf("sourceNetworkId is unknown %d", srcNetworkID) } - results, err := idx.readOnlyQuery.SearchProcesses(context.TODO(), indexerdb.SearchProcessesParams{ + list, err := idx.readOnlyQuery.SearchProcesses(context.TODO(), indexerdb.SearchProcessesParams{ + Offset: int64(offset), + Limit: int64(limit), EntityIDSubstr: entityID, Namespace: int64(namespace), Status: int64(status), SourceNetworkID: int64(srcNetworkID), IDSubstr: strings.ToLower(processID), // we search in lowercase - Offset: int64(offset), - Limit: int64(limit), HaveResults: boolToInt(withResults), FinalResults: boolToInt(finalResults), ManuallyEnded: boolToInt(manuallyEnded), @@ -82,14 +82,24 @@ func (idx *Indexer) ProcessList(limit, offset int, entityID string, processID st if err != nil { return nil, 0, err } - list := [][]byte{} - for _, row := range results { - list = append(list, row.ID) - } - if len(results) == 0 { - return list, 0, nil + count, err := idx.readOnlyQuery.CountProcesses(context.TODO(), indexerdb.CountProcessesParams{ + EntityIDSubstr: entityID, + Namespace: int64(namespace), + Status: int64(status), + SourceNetworkID: int64(srcNetworkID), + IDSubstr: strings.ToLower(processID), // we search in lowercase + HaveResults: boolToInt(withResults), + FinalResults: boolToInt(finalResults), + ManuallyEnded: boolToInt(manuallyEnded), + StartDateAfter: startDateAfter, + StartDateBefore: startDateBefore, + EndDateAfter: endDateAfter, + EndDateBefore: endDateBefore, + }) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, uint64(count), nil } // ProcessExists returns whether the passed processID exists in the db. @@ -107,7 +117,7 @@ func (idx *Indexer) ProcessExists(processID string) bool { // CountTotalProcesses returns the total number of processes indexed. func (idx *Indexer) CountTotalProcesses() uint64 { - count, err := idx.readOnlyQuery.GetProcessCount(context.TODO()) + count, err := idx.readOnlyQuery.CountProcesses(context.TODO(), indexerdb.CountProcessesParams{}) if err != nil { log.Errorf("could not get the process count: %v", err) return 0 diff --git a/vochain/indexer/queries/account.sql b/vochain/indexer/queries/account.sql index c4e570cea..6452f3171 100644 --- a/vochain/indexer/queries/account.sql +++ b/vochain/indexer/queries/account.sql @@ -4,23 +4,28 @@ REPLACE INTO accounts ( ) VALUES (?, ?, ?); -- name: SearchAccounts :many -WITH results AS ( - SELECT * - FROM accounts - WHERE ( - ( - sqlc.arg(account_id_substr) = '' - OR (LENGTH(sqlc.arg(account_id_substr)) = 40 AND LOWER(HEX(account)) = LOWER(sqlc.arg(account_id_substr))) - OR (LENGTH(sqlc.arg(account_id_substr)) < 40 AND INSTR(LOWER(HEX(account)), LOWER(sqlc.arg(account_id_substr))) > 0) - -- TODO: consider keeping an account_hex column for faster searches - ) +SELECT * +FROM accounts +WHERE ( + ( + sqlc.arg(account_id_substr) = '' + OR (LENGTH(sqlc.arg(account_id_substr)) = 40 AND LOWER(HEX(account)) = LOWER(sqlc.arg(account_id_substr))) + OR (LENGTH(sqlc.arg(account_id_substr)) < 40 AND INSTR(LOWER(HEX(account)), LOWER(sqlc.arg(account_id_substr))) > 0) + -- TODO: consider keeping an account_hex column for faster searches ) ) -SELECT *, COUNT(*) OVER() AS total_count -FROM results ORDER BY balance DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); -- name: CountAccounts :one -SELECT COUNT(*) FROM accounts; \ No newline at end of file +SELECT COUNT(*) +FROM accounts +WHERE ( + ( + sqlc.arg(account_id_substr) = '' + OR (LENGTH(sqlc.arg(account_id_substr)) = 40 AND LOWER(HEX(account)) = LOWER(sqlc.arg(account_id_substr))) + OR (LENGTH(sqlc.arg(account_id_substr)) < 40 AND INSTR(LOWER(HEX(account)), LOWER(sqlc.arg(account_id_substr))) > 0) + -- TODO: consider keeping an account_hex column for faster searches + ) +); diff --git a/vochain/indexer/queries/processes.sql b/vochain/indexer/queries/processes.sql index afaca4e40..b14199904 100644 --- a/vochain/indexer/queries/processes.sql +++ b/vochain/indexer/queries/processes.sql @@ -30,52 +30,89 @@ SELECT * FROM processes WHERE id = ? LIMIT 1; +-- name: CountProcesses :one +SELECT COUNT(*) +FROM processes +WHERE ( + LENGTH(sqlc.arg(entity_id_substr)) <= 40 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(entity_id_substr) = '' + OR (LENGTH(sqlc.arg(entity_id_substr)) = 40 AND LOWER(HEX(entity_id)) = LOWER(sqlc.arg(entity_id_substr))) + OR (LENGTH(sqlc.arg(entity_id_substr)) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(sqlc.arg(entity_id_substr))) > 0) + -- TODO: consider keeping an entity_id_hex column for faster searches + ) + AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) + AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) + AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) + AND LENGTH(sqlc.arg(id_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(id_substr) = '' + OR (LENGTH(sqlc.arg(id_substr)) = 64 AND LOWER(HEX(id)) = LOWER(sqlc.arg(id_substr))) + OR (LENGTH(sqlc.arg(id_substr)) < 64 AND INSTR(LOWER(HEX(id)), LOWER(sqlc.arg(id_substr))) > 0) + -- TODO: consider keeping an id_hex column for faster searches + ) + AND ( + sqlc.arg(have_results) = -1 + OR (sqlc.arg(have_results) = 1 AND have_results = TRUE) + OR (sqlc.arg(have_results) = 0 AND have_results = FALSE) + ) + AND ( + sqlc.arg(final_results) = -1 + OR (sqlc.arg(final_results) = 1 AND final_results = TRUE) + OR (sqlc.arg(final_results) = 0 AND final_results = FALSE) + ) + AND ( + sqlc.arg(manually_ended) = -1 + OR (sqlc.arg(manually_ended) = 1 AND manually_ended = TRUE) + OR (sqlc.arg(manually_ended) = 0 AND manually_ended = FALSE) + ) + AND (sqlc.arg(start_date_after) IS NULL OR start_date >= sqlc.arg(start_date_after)) + AND (sqlc.arg(start_date_before) IS NULL OR start_date <= sqlc.arg(start_date_before)) + AND (sqlc.arg(end_date_after) IS NULL OR end_date >= sqlc.arg(end_date_after)) + AND (sqlc.arg(end_date_before) IS NULL OR end_date <= sqlc.arg(end_date_before)) +); + -- name: SearchProcesses :many -WITH results AS ( - SELECT *, - COUNT(*) OVER() AS total_count - FROM processes - WHERE ( - LENGTH(sqlc.arg(entity_id_substr)) <= 40 -- if passed arg is longer, then just abort the query - AND ( - sqlc.arg(entity_id_substr) = '' - OR (LENGTH(sqlc.arg(entity_id_substr)) = 40 AND LOWER(HEX(entity_id)) = LOWER(sqlc.arg(entity_id_substr))) - OR (LENGTH(sqlc.arg(entity_id_substr)) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(sqlc.arg(entity_id_substr))) > 0) - -- TODO: consider keeping an entity_id_hex column for faster searches - ) - AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) - AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) - AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) - AND LENGTH(sqlc.arg(id_substr)) <= 64 -- if passed arg is longer, then just abort the query - AND ( - sqlc.arg(id_substr) = '' - OR (LENGTH(sqlc.arg(id_substr)) = 64 AND LOWER(HEX(id)) = LOWER(sqlc.arg(id_substr))) - OR (LENGTH(sqlc.arg(id_substr)) < 64 AND INSTR(LOWER(HEX(id)), LOWER(sqlc.arg(id_substr))) > 0) - -- TODO: consider keeping an id_hex column for faster searches - ) - AND ( - sqlc.arg(have_results) = -1 - OR (sqlc.arg(have_results) = 1 AND have_results = TRUE) - OR (sqlc.arg(have_results) = 0 AND have_results = FALSE) - ) - AND ( - sqlc.arg(final_results) = -1 - OR (sqlc.arg(final_results) = 1 AND final_results = TRUE) - OR (sqlc.arg(final_results) = 0 AND final_results = FALSE) - ) - AND ( - sqlc.arg(manually_ended) = -1 - OR (sqlc.arg(manually_ended) = 1 AND manually_ended = TRUE) - OR (sqlc.arg(manually_ended) = 0 AND manually_ended = FALSE) - ) - AND (sqlc.arg(start_date_after) IS NULL OR start_date >= sqlc.arg(start_date_after)) - AND (sqlc.arg(start_date_before) IS NULL OR start_date <= sqlc.arg(start_date_before)) - AND (sqlc.arg(end_date_after) IS NULL OR end_date >= sqlc.arg(end_date_after)) - AND (sqlc.arg(end_date_before) IS NULL OR end_date <= sqlc.arg(end_date_before)) +SELECT id +FROM processes +WHERE ( + LENGTH(sqlc.arg(entity_id_substr)) <= 40 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(entity_id_substr) = '' + OR (LENGTH(sqlc.arg(entity_id_substr)) = 40 AND LOWER(HEX(entity_id)) = LOWER(sqlc.arg(entity_id_substr))) + OR (LENGTH(sqlc.arg(entity_id_substr)) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(sqlc.arg(entity_id_substr))) > 0) + -- TODO: consider keeping an entity_id_hex column for faster searches + ) + AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) + AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) + AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) + AND LENGTH(sqlc.arg(id_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(id_substr) = '' + OR (LENGTH(sqlc.arg(id_substr)) = 64 AND LOWER(HEX(id)) = LOWER(sqlc.arg(id_substr))) + OR (LENGTH(sqlc.arg(id_substr)) < 64 AND INSTR(LOWER(HEX(id)), LOWER(sqlc.arg(id_substr))) > 0) + -- TODO: consider keeping an id_hex column for faster searches + ) + AND ( + sqlc.arg(have_results) = -1 + OR (sqlc.arg(have_results) = 1 AND have_results = TRUE) + OR (sqlc.arg(have_results) = 0 AND have_results = FALSE) + ) + AND ( + sqlc.arg(final_results) = -1 + OR (sqlc.arg(final_results) = 1 AND final_results = TRUE) + OR (sqlc.arg(final_results) = 0 AND final_results = FALSE) ) + AND ( + sqlc.arg(manually_ended) = -1 + OR (sqlc.arg(manually_ended) = 1 AND manually_ended = TRUE) + OR (sqlc.arg(manually_ended) = 0 AND manually_ended = FALSE) + ) + AND (sqlc.arg(start_date_after) IS NULL OR start_date >= sqlc.arg(start_date_after)) + AND (sqlc.arg(start_date_before) IS NULL OR start_date <= sqlc.arg(start_date_before)) + AND (sqlc.arg(end_date_after) IS NULL OR end_date >= sqlc.arg(end_date_after)) + AND (sqlc.arg(end_date_before) IS NULL OR end_date <= sqlc.arg(end_date_before)) ) -SELECT id, total_count -FROM results ORDER BY creation_time DESC, id ASC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); @@ -125,9 +162,6 @@ UPDATE processes SET vote_count = (SELECT COUNT(*) FROM votes WHERE process_id = id) WHERE id = sqlc.arg(id); --- name: GetProcessCount :one -SELECT COUNT(*) FROM processes; - -- name: GetEntityCount :one SELECT COUNT(DISTINCT entity_id) FROM processes; diff --git a/vochain/indexer/queries/token_fees.sql b/vochain/indexer/queries/token_fees.sql index 2a46393db..5b6cd0b64 100644 --- a/vochain/indexer/queries/token_fees.sql +++ b/vochain/indexer/queries/token_fees.sql @@ -7,18 +7,23 @@ INSERT INTO token_fees ( ?, ?, ? ); +-- name: CountTokenFees :one +SELECT COUNT(*) +FROM token_fees +WHERE ( + (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account))) + AND (sqlc.arg(tx_type) = '' OR LOWER(tx_type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(reference) = '' OR LOWER(reference) = LOWER(sqlc.arg(reference))) +); + -- name: SearchTokenFees :many -WITH results AS ( - SELECT * - FROM token_fees - WHERE ( - (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account))) - AND (sqlc.arg(tx_type) = '' OR LOWER(tx_type) = LOWER(sqlc.arg(tx_type))) - AND (sqlc.arg(reference) = '' OR LOWER(reference) = LOWER(sqlc.arg(reference))) - ) +SELECT * +FROM token_fees +WHERE ( + (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account))) + AND (sqlc.arg(tx_type) = '' OR LOWER(tx_type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(reference) = '' OR LOWER(reference) = LOWER(sqlc.arg(reference))) ) -SELECT *, COUNT(*) OVER() AS total_count -FROM results ORDER BY spend_time DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/queries/token_transfers.sql b/vochain/indexer/queries/token_transfers.sql index fdc6cee08..01eeb786f 100644 --- a/vochain/indexer/queries/token_transfers.sql +++ b/vochain/indexer/queries/token_transfers.sql @@ -12,21 +12,29 @@ SELECT * FROM token_transfers WHERE tx_hash = ? LIMIT 1; +-- name: CountTokenTransfers :one +SELECT COUNT(*) +FROM token_transfers +WHERE ( + (sqlc.arg(from_or_to_account) = '' + OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_or_to_account)) + OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(from_or_to_account)) + ) + AND (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account))) + AND (sqlc.arg(to_account) = '' OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(to_account))) +); + -- name: SearchTokenTransfers :many -WITH results AS ( - SELECT * - FROM token_transfers - WHERE ( - (sqlc.arg(from_or_to_account) = '' OR ( - LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_or_to_account)) +SELECT * +FROM token_transfers +WHERE ( + (sqlc.arg(from_or_to_account) = '' + OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_or_to_account)) OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(from_or_to_account)) - )) - AND (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account))) - AND (sqlc.arg(to_account) = '' OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(to_account))) - ) + ) + AND (sqlc.arg(from_account) = '' OR LOWER(HEX(from_account)) = LOWER(sqlc.arg(from_account))) + AND (sqlc.arg(to_account) = '' OR LOWER(HEX(to_account)) = LOWER(sqlc.arg(to_account))) ) -SELECT *, COUNT(*) OVER() AS total_count -FROM results ORDER BY transfer_time DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index c9c152482..24759c361 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -20,7 +20,20 @@ WHERE hash = ? LIMIT 1; -- name: CountTransactions :one -SELECT COUNT(*) FROM transactions; +SELECT COUNT(*) +FROM transactions +WHERE ( + (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) + AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(tx_subtype) = '' OR LOWER(subtype) = LOWER(sqlc.arg(tx_subtype))) + AND (sqlc.arg(tx_signer) = '' OR LOWER(HEX(signer)) = LOWER(sqlc.arg(tx_signer))) + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) +); -- name: CountTransactionsByHeight :one SELECT COUNT(*) FROM transactions @@ -32,7 +45,7 @@ WHERE block_height = ? AND block_index = ? LIMIT 1; -- name: SearchTransactions :many -SELECT *, COUNT(*) OVER() AS total_count +SELECT * FROM transactions WHERE (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) diff --git a/vochain/indexer/queries/votes.sql b/vochain/indexer/queries/votes.sql index 9e92afc7d..414d143a2 100644 --- a/vochain/indexer/queries/votes.sql +++ b/vochain/indexer/queries/votes.sql @@ -20,34 +20,47 @@ WHERE v.nullifier = ? LIMIT 1; -- name: CountVotes :one -SELECT COUNT(*) FROM votes; +SELECT COUNT(*) +FROM votes +WHERE ( + LENGTH(sqlc.arg(process_id_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(process_id_substr) = '' + OR (LENGTH(sqlc.arg(process_id_substr)) = 64 AND LOWER(HEX(process_id)) = LOWER(sqlc.arg(process_id_substr))) + OR (LENGTH(sqlc.arg(process_id_substr)) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(sqlc.arg(process_id_substr))) > 0) + -- TODO: consider keeping an process_id_hex column for faster searches + ) + AND LENGTH(sqlc.arg(nullifier_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(nullifier_substr) = '' + OR (LENGTH(sqlc.arg(nullifier_substr)) = 64 AND LOWER(HEX(nullifier)) = LOWER(sqlc.arg(nullifier_substr))) + OR (LENGTH(sqlc.arg(nullifier_substr)) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(sqlc.arg(nullifier_substr))) > 0) + -- TODO: consider keeping an nullifier_hex column for faster searches + ) +); -- name: SearchVotes :many -WITH results AS ( - SELECT v.*, t.hash - FROM votes AS v - LEFT JOIN transactions AS t - ON v.block_height = t.block_height - AND v.block_index = t.block_index - WHERE ( - LENGTH(sqlc.arg(process_id_substr)) <= 64 -- if passed arg is longer, then just abort the query - AND ( - sqlc.arg(process_id_substr) = '' - OR (LENGTH(sqlc.arg(process_id_substr)) = 64 AND LOWER(HEX(process_id)) = LOWER(sqlc.arg(process_id_substr))) - OR (LENGTH(sqlc.arg(process_id_substr)) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(sqlc.arg(process_id_substr))) > 0) - -- TODO: consider keeping an process_id_hex column for faster searches - ) - AND LENGTH(sqlc.arg(nullifier_substr)) <= 64 -- if passed arg is longer, then just abort the query - AND ( - sqlc.arg(nullifier_substr) = '' - OR (LENGTH(sqlc.arg(nullifier_substr)) = 64 AND LOWER(HEX(nullifier)) = LOWER(sqlc.arg(nullifier_substr))) - OR (LENGTH(sqlc.arg(nullifier_substr)) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(sqlc.arg(nullifier_substr))) > 0) - -- TODO: consider keeping an nullifier_hex column for faster searches - ) +SELECT v.*, t.hash +FROM votes AS v +LEFT JOIN transactions AS t + ON v.block_height = t.block_height + AND v.block_index = t.block_index +WHERE ( + LENGTH(sqlc.arg(process_id_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(process_id_substr) = '' + OR (LENGTH(sqlc.arg(process_id_substr)) = 64 AND LOWER(HEX(process_id)) = LOWER(sqlc.arg(process_id_substr))) + OR (LENGTH(sqlc.arg(process_id_substr)) < 64 AND INSTR(LOWER(HEX(process_id)), LOWER(sqlc.arg(process_id_substr))) > 0) + -- TODO: consider keeping an process_id_hex column for faster searches + ) + AND LENGTH(sqlc.arg(nullifier_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(nullifier_substr) = '' + OR (LENGTH(sqlc.arg(nullifier_substr)) = 64 AND LOWER(HEX(nullifier)) = LOWER(sqlc.arg(nullifier_substr))) + OR (LENGTH(sqlc.arg(nullifier_substr)) < 64 AND INSTR(LOWER(HEX(nullifier)), LOWER(sqlc.arg(nullifier_substr))) > 0) + -- TODO: consider keeping an nullifier_hex column for faster searches ) ) -SELECT *, COUNT(*) OVER() AS total_count -FROM results -ORDER BY block_height DESC, nullifier ASC +ORDER BY v.block_height DESC, v.nullifier ASC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 41a6a1818..850822be2 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -21,7 +21,7 @@ var ErrTransactionNotFound = fmt.Errorf("transaction not found") // CountTotalTransactions returns the number of transactions indexed func (idx *Indexer) CountTotalTransactions() (uint64, error) { - count, err := idx.readOnlyQuery.CountTransactions(context.TODO()) + count, err := idx.readOnlyQuery.CountTransactions(context.TODO(), indexerdb.CountTransactionsParams{}) return uint64(count), err } @@ -69,11 +69,11 @@ func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64 return indexertypes.TransactionFromDB(&sqlTxRef), nil } -// SearchTransactions returns the list of transactions indexed. +// TransactionList returns the list of transactions indexed. // blockHeight, hash, txType, txSubtype and txSigner are optional, if declared as zero-value will be ignored. // hash matches substrings. // The first one returned is the newest, so they are in descending order. -func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txHash, txType, txSubtype, txSigner string) ([]*indexertypes.TransactionMetadata, uint64, error) { +func (idx *Indexer) TransactionList(limit, offset int, blockHeight uint64, txHash, txType, txSubtype, txSigner string) ([]*indexertypes.TransactionMetadata, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } @@ -94,12 +94,19 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx } list := []*indexertypes.TransactionMetadata{} for _, row := range results { - list = append(list, indexertypes.TransactionMetadataFromDBRow(&row)) + list = append(list, indexertypes.TransactionMetadataFromDB(&row)) } - if len(results) == 0 { - return list, 0, nil + count, err := idx.readOnlyQuery.CountTransactions(context.TODO(), indexerdb.CountTransactionsParams{ + HashSubstr: txHash, + BlockHeight: blockHeight, + TxType: txType, + TxSubtype: txSubtype, + TxSigner: txSigner, + }) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, uint64(count), nil } func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { diff --git a/vochain/indexer/vote.go b/vochain/indexer/vote.go index ca66e59d3..5c0986a8f 100644 --- a/vochain/indexer/vote.go +++ b/vochain/indexer/vote.go @@ -91,15 +91,19 @@ func (idx *Indexer) VoteList(limit, offset int, processID string, nullifier stri } list = append(list, envelopeMetadata) } - if len(results) == 0 { - return list, 0, nil + count, err := idx.readOnlyQuery.CountVotes(context.TODO(), indexerdb.CountVotesParams{ + ProcessIDSubstr: processID, + NullifierSubstr: strings.ToLower(nullifier), // we search in lowercase + }) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, uint64(count), nil } // CountTotalVotes returns the total number of envelopes. func (idx *Indexer) CountTotalVotes() (uint64, error) { - height, err := idx.readOnlyQuery.CountVotes(context.TODO()) + height, err := idx.readOnlyQuery.CountVotes(context.TODO(), indexerdb.CountVotesParams{}) return uint64(height), err }