From 43d20894d6f619ddb1835272cd480759a2920bc7 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 28 Aug 2024 14:43:42 +0200 Subject: [PATCH 1/2] api: endpoint /elections now filters by title and description (WIP) --- api/api.go | 2 + api/api_types.go | 4 ++ api/elections.go | 8 +++ vochain/indexer/db/models.go | 2 + vochain/indexer/db/processes.sql.go | 62 ++++++++++++------- vochain/indexer/indexer_test.go | 34 +++++----- ...0013_add_process_title_and_description.sql | 7 +++ vochain/indexer/process.go | 5 +- vochain/indexer/queries/processes.sql | 6 ++ 9 files changed, 89 insertions(+), 41 deletions(-) create mode 100644 vochain/indexer/migrations/0013_add_process_title_and_description.sql diff --git a/api/api.go b/api/api.go index 33fcba409..fe71a4a7f 100644 --- a/api/api.go +++ b/api/api.go @@ -67,6 +67,8 @@ const ( ParamVoteId = "voteId" ParamPage = "page" ParamLimit = "limit" + ParamTitle = "title" + ParamDescription = "description" ParamStatus = "status" ParamWithResults = "withResults" ParamFinalResults = "finalResults" diff --git a/api/api_types.go b/api/api_types.go index efb390503..bc0229d0e 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -33,6 +33,8 @@ type ElectionParams struct { StartDateBefore *time.Time `json:"startDateBefore,omitempty"` EndDateAfter *time.Time `json:"endDateAfter,omitempty"` EndDateBefore *time.Time `json:"endDateBefore,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` } // OrganizationParams allows the client to filter organizations @@ -122,6 +124,8 @@ type ElectionSummary struct { Results [][]*types.BigInt `json:"result,omitempty"` ManuallyEnded bool `json:"manuallyEnded"` ChainID string `json:"chainId"` + Title string `json:"title"` + Description string `json:"description"` } // ElectionsList is used to return a paginated list to the client diff --git a/api/elections.go b/api/elections.go index 9e921f616..2f3ceece1 100644 --- a/api/elections.go +++ b/api/elections.go @@ -256,6 +256,8 @@ func (a *API) electionListByPageHandler(_ *apirest.APIdata, ctx *httprouter.HTTP // @Param organizationId query string false "Filter by partial organizationId" // @Param status query string false "Election status" Enums(ready, paused, canceled, ended, results) // @Param electionId query string false "Filter by partial electionId" +// @Param title query string false "Filter by election title" +// @Param descrition query string false "Filter by election description" // @Param withResults query boolean false "Filter by (partial or final) results available or not" // @Param finalResults query boolean false "Filter by final results available or not" // @Param manuallyEnded query boolean false "Filter by whether the election was manually ended or not" @@ -268,6 +270,8 @@ func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContex ParamStatus, ParamOrganizationId, ParamElectionId, + ParamTitle, + ParamDescription, ParamWithResults, ParamFinalResults, ParamManuallyEnded, @@ -316,6 +320,8 @@ func (a *API) electionList(params *ElectionParams) (*ElectionsList, error) { params.StartDateBefore, params.EndDateAfter, params.EndDateBefore, + params.Title, + params.Description, ) if err != nil { return nil, ErrIndexerQueryFailed.WithErr(err) @@ -831,6 +837,8 @@ func electionParams(f func(key string) string, keys ...string) (*ElectionParams, PaginationParams: pagination, OrganizationID: util.TrimHex(strings[ParamOrganizationId]), ElectionID: util.TrimHex(strings[ParamElectionId]), + Title: strings[ParamTitle], + Description: strings[ParamDescription], Status: strings[ParamStatus], WithResults: bools[ParamWithResults], FinalResults: bools[ParamFinalResults], diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 08a6e0e2b..6c65131b5 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -45,6 +45,8 @@ type Process struct { SourceBlockHeight int64 SourceNetworkID int64 ManuallyEnded bool + Title string + Description string } type TokenTransfer struct { diff --git a/vochain/indexer/db/processes.sql.go b/vochain/indexer/db/processes.sql.go index b274c802c..477d50df2 100644 --- a/vochain/indexer/db/processes.sql.go +++ b/vochain/indexer/db/processes.sql.go @@ -26,6 +26,7 @@ func (q *Queries) ComputeProcessVoteCount(ctx context.Context, id types.ProcessI const createProcess = `-- name: CreateProcess :execresult INSERT INTO processes ( id, entity_id, start_date, end_date, manually_ended, + title, description, vote_count, have_results, final_results, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, @@ -38,6 +39,7 @@ INSERT INTO processes ( results_votes, results_weight, results_block_height ) VALUES ( ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, @@ -57,6 +59,8 @@ type CreateProcessParams struct { StartDate time.Time EndDate time.Time ManuallyEnded bool + Title string + Description string VoteCount int64 HaveResults bool FinalResults bool @@ -87,6 +91,8 @@ func (q *Queries) CreateProcess(ctx context.Context, arg CreateProcessParams) (s arg.StartDate, arg.EndDate, arg.ManuallyEnded, + arg.Title, + arg.Description, arg.VoteCount, arg.HaveResults, arg.FinalResults, @@ -123,7 +129,7 @@ func (q *Queries) GetEntityCount(ctx context.Context) (int64, error) { } const getProcess = `-- name: GetProcess :one -SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended FROM processes +SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, title, description FROM processes WHERE id = ? LIMIT 1 ` @@ -160,6 +166,8 @@ func (q *Queries) GetProcess(ctx context.Context, id types.ProcessID) (Process, &i.SourceBlockHeight, &i.SourceNetworkID, &i.ManuallyEnded, + &i.Title, + &i.Description, ) return i, err } @@ -218,7 +226,7 @@ func (q *Queries) GetProcessStatus(ctx context.Context, id types.ProcessID) (int const searchEntities = `-- name: SearchEntities :many WITH results AS ( - SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended + SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, title, description FROM processes WHERE (?3 = '' OR (INSTR(LOWER(HEX(entity_id)), ?3) > 0)) ) @@ -269,7 +277,7 @@ func (q *Queries) SearchEntities(ctx context.Context, arg SearchEntitiesParams) const searchProcesses = `-- name: SearchProcesses :many WITH results AS ( - SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, + SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, title, description, COUNT(*) OVER() AS total_count FROM processes WHERE ( @@ -280,35 +288,39 @@ WITH results AS ( OR (LENGTH(?3) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(?3)) > 0) -- TODO: consider keeping an entity_id_hex column for faster searches ) - AND (?4 = 0 OR namespace = ?4) - AND (?5 = 0 OR status = ?5) - AND (?6 = 0 OR source_network_id = ?6) - AND LENGTH(?7) <= 64 -- if passed arg is longer, then just abort the query + -- TODO: replace this simple INSTR of title and description with something better + -- like https://www.sqlite.org/fts5.html + AND (?4 = '' OR INSTR(LOWER(title), LOWER(?4)) > 0) + AND (?5 = '' OR INSTR(LOWER(description), LOWER(?5)) > 0) + AND (?6 = 0 OR namespace = ?6) + AND (?7 = 0 OR status = ?7) + AND (?8 = 0 OR source_network_id = ?8) + AND LENGTH(?9) <= 64 -- if passed arg is longer, then just abort the query AND ( - ?7 = '' - OR (LENGTH(?7) = 64 AND LOWER(HEX(id)) = LOWER(?7)) - OR (LENGTH(?7) < 64 AND INSTR(LOWER(HEX(id)), LOWER(?7)) > 0) + ?9 = '' + OR (LENGTH(?9) = 64 AND LOWER(HEX(id)) = LOWER(?9)) + OR (LENGTH(?9) < 64 AND INSTR(LOWER(HEX(id)), LOWER(?9)) > 0) -- TODO: consider keeping an id_hex column for faster searches ) AND ( - ?8 = -1 - OR (?8 = 1 AND have_results = TRUE) - OR (?8 = 0 AND have_results = FALSE) + ?10 = -1 + OR (?10 = 1 AND have_results = TRUE) + OR (?10 = 0 AND have_results = FALSE) ) AND ( - ?9 = -1 - OR (?9 = 1 AND final_results = TRUE) - OR (?9 = 0 AND final_results = FALSE) + ?11 = -1 + OR (?11 = 1 AND final_results = TRUE) + OR (?11 = 0 AND final_results = FALSE) ) AND ( - ?10 = -1 - OR (?10 = 1 AND manually_ended = TRUE) - OR (?10 = 0 AND manually_ended = FALSE) + ?12 = -1 + OR (?12 = 1 AND manually_ended = TRUE) + OR (?12 = 0 AND manually_ended = FALSE) ) - AND (?11 IS NULL OR start_date >= ?11) - AND (?12 IS NULL OR start_date <= ?12) - AND (?13 IS NULL OR end_date >= ?13) - AND (?14 IS NULL OR end_date <= ?14) + AND (?13 IS NULL OR start_date >= ?13) + AND (?14 IS NULL OR start_date <= ?14) + AND (?15 IS NULL OR end_date >= ?15) + AND (?16 IS NULL OR end_date <= ?16) ) ) SELECT id, total_count @@ -322,6 +334,8 @@ type SearchProcessesParams struct { Offset int64 Limit int64 EntityIDSubstr interface{} + Title interface{} + Description interface{} Namespace interface{} Status interface{} SourceNetworkID interface{} @@ -345,6 +359,8 @@ func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams arg.Offset, arg.Limit, arg.EntityIDSubstr, + arg.Title, + arg.Description, arg.Namespace, arg.Status, arg.SourceNetworkID, diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index fad19d248..7fdb139e3 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -332,7 +332,7 @@ func testProcessList(t *testing.T, procsCount int) { for len(procs) < procsCount { fmt.Printf("%x\n", eidProcsCount) fmt.Printf("%s\n", hex.EncodeToString(eidProcsCount)) - list, total, err := idx.ProcessList(10, last, hex.EncodeToString(eidProcsCount), "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, total, err := idx.ProcessList(10, last, hex.EncodeToString(eidProcsCount), "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { t.Fatal(err) } @@ -351,7 +351,7 @@ func testProcessList(t *testing.T, procsCount int) { } qt.Assert(t, procs, qt.HasLen, procsCount) - _, total, err := idx.ProcessList(64, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + _, total, err := idx.ProcessList(64, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, total, qt.Equals, uint64(10+procsCount)) @@ -369,7 +369,7 @@ func testProcessList(t *testing.T, procsCount int) { qt.Assert(t, countEntityProcs([]byte("not an entity id that exists")), qt.Equals, int64(-1)) // Past the end (from=10000) should return an empty list - emptyList, _, err := idx.ProcessList(64, 10000, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + emptyList, _, err := idx.ProcessList(64, 10000, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, emptyList, qt.DeepEquals, [][]byte{}) } @@ -459,7 +459,7 @@ func TestProcessSearch(t *testing.T) { app.AdvanceTestBlock() // Exact process search - list, _, err := idx.ProcessList(10, 0, hex.EncodeToString(eidTest), pidExact, 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err := idx.ProcessList(10, 0, hex.EncodeToString(eidTest), pidExact, 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { t.Fatal(err) } @@ -468,7 +468,7 @@ func TestProcessSearch(t *testing.T) { } // Exact process search, with it being encrypted. // This once caused a sqlite bug due to a mistake in the SQL query. - list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), pidExactEncrypted, 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), pidExactEncrypted, 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { t.Fatal(err) } @@ -477,7 +477,7 @@ func TestProcessSearch(t *testing.T) { } // Search for nonexistent process list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), - "4011d50537fa164b6fef261141797bbe4014526f", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + "4011d50537fa164b6fef261141797bbe4014526f", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { t.Fatal(err) } @@ -486,7 +486,7 @@ func TestProcessSearch(t *testing.T) { } // Search containing part of all manually-defined processes list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), - "011d50537fa164b6fef261141797bbe4014526e", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + "011d50537fa164b6fef261141797bbe4014526e", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { t.Fatal(err) } @@ -495,7 +495,7 @@ func TestProcessSearch(t *testing.T) { } list, _, err = idx.ProcessList(100, 0, hex.EncodeToString(eidTest), - "0c6ca22d2c175a1fbdd15d7595ae532bb1094b5", 0, 0, models.ProcessStatus_ENDED, nil, nil, nil, nil, nil, nil, nil) + "0c6ca22d2c175a1fbdd15d7595ae532bb1094b5", 0, 0, models.ProcessStatus_ENDED, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { t.Fatal(err) } @@ -504,17 +504,17 @@ func TestProcessSearch(t *testing.T) { } // Partial process search as uppercase hex - list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), "011D50537FA164B6FEF261141797BBE4014526E", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), "011D50537FA164B6FEF261141797BBE4014526E", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, len(processIds)) // Partial process search as mixed case hex - list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), "011D50537fA164B6FeF261141797BbE4014526E", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(10, 0, hex.EncodeToString(eidTest), "011D50537fA164B6FeF261141797BbE4014526E", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, len(processIds)) // Search with an exact Entity ID, but starting with a null byte. // This can trip up sqlite, as it assumes TEXT strings are NUL-terminated. - list, _, err = idx.ProcessList(100, 0, "\x00foobar", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(100, 0, "\x00foobar", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { t.Fatal(err) } @@ -523,12 +523,12 @@ func TestProcessSearch(t *testing.T) { } // list all processes, with a max of 10 - list, _, err = idx.ProcessList(10, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(10, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, 10) // list all processes, with a max of 1000 - list, _, err = idx.ProcessList(1000, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(1000, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, 21) } @@ -577,25 +577,25 @@ func TestProcessListWithNamespaceAndStatus(t *testing.T) { app.AdvanceTestBlock() // Get the process list for namespace 123 - list, _, err := idx.ProcessList(100, 0, hex.EncodeToString(eid20), "", 123, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err := idx.ProcessList(100, 0, hex.EncodeToString(eid20), "", 123, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) // Check there are exactly 10 qt.Assert(t, len(list), qt.CmpEquals(), 10) // Get the process list for all namespaces - list, _, err = idx.ProcessList(100, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(100, 0, "", "", 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) // Check there are exactly 10 + 10 qt.Assert(t, len(list), qt.CmpEquals(), 20) // Get the process list for namespace 10 - list, _, err = idx.ProcessList(100, 0, "", "", 10, 0, 0, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(100, 0, "", "", 10, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) // Check there is exactly 1 qt.Assert(t, len(list), qt.CmpEquals(), 1) // Get the process list for namespace 10 - list, _, err = idx.ProcessList(100, 0, "", "", 0, 0, models.ProcessStatus_READY, nil, nil, nil, nil, nil, nil, nil) + list, _, err = idx.ProcessList(100, 0, "", "", 0, 0, models.ProcessStatus_READY, nil, nil, nil, nil, nil, nil, nil, "", "") qt.Assert(t, err, qt.IsNil) // Check there is exactly 1 qt.Assert(t, len(list), qt.CmpEquals(), 10) diff --git a/vochain/indexer/migrations/0013_add_process_title_and_description.sql b/vochain/indexer/migrations/0013_add_process_title_and_description.sql new file mode 100644 index 000000000..57aac26c4 --- /dev/null +++ b/vochain/indexer/migrations/0013_add_process_title_and_description.sql @@ -0,0 +1,7 @@ +-- +goose Up +ALTER TABLE processes ADD COLUMN title TEXT NOT NULL DEFAULT ''; +ALTER TABLE processes ADD COLUMN description TEXT NOT NULL DEFAULT ''; + +-- +goose Down +ALTER TABLE processes DROP COLUMN description; +ALTER TABLE processes DROP COLUMN title; diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go index 3a33fee32..ed9c0d881 100644 --- a/vochain/indexer/process.go +++ b/vochain/indexer/process.go @@ -52,6 +52,7 @@ func (idx *Indexer) ProcessList(limit, offset int, entityID string, processID st namespace uint32, srcNetworkID int32, status models.ProcessStatus, withResults, finalResults, manuallyEnded *bool, startDateAfter, startDateBefore, endDateAfter, endDateBefore *time.Time, + title, description string, ) ([][]byte, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) @@ -78,6 +79,8 @@ func (idx *Indexer) ProcessList(limit, offset int, entityID string, processID st StartDateBefore: startDateBefore, EndDateAfter: endDateAfter, EndDateBefore: endDateBefore, + Title: title, + Description: description, }) if err != nil { return nil, 0, err @@ -98,7 +101,7 @@ func (idx *Indexer) ProcessExists(processID string) bool { if len(processID) != 64 { return false } - _, count, err := idx.ProcessList(1, 0, "", processID, 0, 0, 0, nil, nil, nil, nil, nil, nil, nil) + _, count, err := idx.ProcessList(1, 0, "", processID, 0, 0, 0, nil, nil, nil, nil, nil, nil, nil, "", "") if err != nil { log.Errorw(err, "indexer query failed") } diff --git a/vochain/indexer/queries/processes.sql b/vochain/indexer/queries/processes.sql index afaca4e40..9373195e0 100644 --- a/vochain/indexer/queries/processes.sql +++ b/vochain/indexer/queries/processes.sql @@ -1,6 +1,7 @@ -- name: CreateProcess :execresult INSERT INTO processes ( id, entity_id, start_date, end_date, manually_ended, + title, description, vote_count, have_results, final_results, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, @@ -13,6 +14,7 @@ INSERT INTO processes ( results_votes, results_weight, results_block_height ) VALUES ( ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, @@ -43,6 +45,10 @@ WITH results AS ( OR (LENGTH(sqlc.arg(entity_id_substr)) < 40 AND INSTR(LOWER(HEX(entity_id)), LOWER(sqlc.arg(entity_id_substr))) > 0) -- TODO: consider keeping an entity_id_hex column for faster searches ) + -- TODO: replace this simple INSTR of title and description with something better + -- like https://www.sqlite.org/fts5.html + AND (sqlc.arg(title) = '' OR INSTR(LOWER(title), LOWER(sqlc.arg(title))) > 0) + AND (sqlc.arg(description) = '' OR INSTR(LOWER(description), LOWER(sqlc.arg(description))) > 0) AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) From f72f3d405f6a3674c63031025174941bc393d09e Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 28 Aug 2024 15:36:14 +0200 Subject: [PATCH 2/2] half-baked attempt at indexing title and description after offchain download happens --- service/offchaindata.go | 1 + vochain/indexer/process.go | 29 +++++++++++++++++++ vochain/offchaindatahandler/metadata.go | 11 ++++--- .../offchaindatahandler.go | 10 +++++-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/service/offchaindata.go b/service/offchaindata.go index 18a5d4781..f3a73a34b 100644 --- a/service/offchaindata.go +++ b/service/offchaindata.go @@ -32,6 +32,7 @@ func (vs *VocdoniService) OffChainDataHandler() error { } vs.OffChainData = offchaindatahandler.NewOffChainDataHandler( vs.App, + vs.Indexer, vs.DataDownloader, vs.CensusDB, vs.Config.SkipPreviousOffchainData, diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go index ed9c0d881..00c42469b 100644 --- a/vochain/indexer/process.go +++ b/vochain/indexer/process.go @@ -3,6 +3,7 @@ package indexer import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "strings" @@ -10,6 +11,8 @@ import ( "go.vocdoni.io/proto/build/go/models" + "go.vocdoni.io/dvote/api" + "go.vocdoni.io/dvote/data" "go.vocdoni.io/dvote/log" indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" @@ -315,3 +318,29 @@ func (idx *Indexer) updateProcess(ctx context.Context, queries *indexerdb.Querie } return nil } + +// updateProcessMetadata synchronize title and description +// with the information fetched over IPFS. +func (idx *Indexer) UpdateProcessMetadata(d data.Storage, pid []byte, uri string) error { + // Try to retrieve the election metadata + if d != nil { + stgCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + // TODO: abstract all of this somewhere else and just call from here + metadataBytes, err := d.Retrieve(stgCtx, uri, MaxOffchainFileSize) + if err != nil { + log.Warnf("cannot get metadata from %s: %v", election.MetadataURL, err) + } else { + // if metadata exists and is not encrypted, add it to the indexed election + // if the metadata is not encrypted, unmarshal it, otherwise store it as bytes + if !election.ElectionMode.EncryptedMetaData { + electionMetadata := api.ElectionMetadata{} + if err := json.Unmarshal(metadataBytes, &electionMetadata); err != nil { + log.Warnf("cannot unmarshal metadata from %s: %v", election.MetadataURL, err) + } + election.Metadata = &electionMetadata + } + } + } +} diff --git a/vochain/offchaindatahandler/metadata.go b/vochain/offchaindatahandler/metadata.go index 9e1738ad2..1f8603f62 100644 --- a/vochain/offchaindatahandler/metadata.go +++ b/vochain/offchaindatahandler/metadata.go @@ -8,12 +8,15 @@ import ( // enqueueMetadata enqueue a election metadata for download. // (safe for concurrent use, simply pushes an item to a channel) -func (d *OffChainDataHandler) enqueueMetadata(uri string) { - if !strings.HasPrefix(uri, d.storage.RemoteStorage.URIprefix()) { - log.Warnf("metadata URI not valid: %s", uri) +func (d *OffChainDataHandler) enqueueMetadata(item importItem) { + if !strings.HasPrefix(item.uri, d.storage.RemoteStorage.URIprefix()) { + log.Warnf("metadata URI not valid: %s", item.uri) return } - d.storage.AddToQueue(uri, func(s string, b []byte) { + d.storage.AddToQueue(item.uri, func(s string, b []byte) { log.Infof("metadata downloaded successfully from %s (%d bytes)", s, len(b)) + if item.itemType == itemTypeElectionMetadata && len(item.pid) > 0 { + d.indexer.UpdateProcessMetadata(d.storage.RemoteStorage, item.pid, item.uri) + } }, true) } diff --git a/vochain/offchaindatahandler/offchaindatahandler.go b/vochain/offchaindatahandler/offchaindatahandler.go index d6e97bd9b..86270d2fa 100644 --- a/vochain/offchaindatahandler/offchaindatahandler.go +++ b/vochain/offchaindatahandler/offchaindatahandler.go @@ -8,8 +8,10 @@ import ( "go.vocdoni.io/dvote/api/censusdb" "go.vocdoni.io/dvote/data/downloader" "go.vocdoni.io/dvote/log" + "go.vocdoni.io/dvote/types" "go.vocdoni.io/dvote/util" "go.vocdoni.io/dvote/vochain" + "go.vocdoni.io/dvote/vochain/indexer" "go.vocdoni.io/dvote/vochain/state" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" @@ -26,6 +28,7 @@ type importItem struct { itemType int uri string censusRoot string + pid types.HexBytes } var itemTypesToString = map[int]string{ @@ -42,6 +45,7 @@ var itemTypesToString = map[int]string{ // offchain data (usually on IPFS). type OffChainDataHandler struct { vochain *vochain.BaseApplication + indexer *indexer.Indexer census *censusdb.CensusDB storage *downloader.Downloader queue []importItem @@ -52,11 +56,12 @@ type OffChainDataHandler struct { // NewOffChainDataHandler creates a new instance of the off chain data downloader daemon. // It will subscribe to Vochain events and perform data import. -func NewOffChainDataHandler(v *vochain.BaseApplication, d *downloader.Downloader, +func NewOffChainDataHandler(v *vochain.BaseApplication, i *indexer.Indexer, d *downloader.Downloader, c *censusdb.CensusDB, importOnlyNew bool, ) *OffChainDataHandler { od := OffChainDataHandler{ vochain: v, + indexer: i, census: c, storage: d, importOnlyNew: importOnlyNew, @@ -87,7 +92,7 @@ func (d *OffChainDataHandler) Commit(_ uint32) error { go d.enqueueOffchainCensus(item.censusRoot, item.uri) case itemTypeElectionMetadata, itemTypeAccountMetadata: log.Infow("importing data", "type", itemTypesToString[item.itemType], "uri", item.uri) - go d.enqueueMetadata(item.uri) + go d.enqueueMetadata(item) default: log.Errorf("unknown import item %d", item.itemType) } @@ -107,6 +112,7 @@ func (d *OffChainDataHandler) OnProcess(p *models.Process, _ int32) { if m := p.GetMetadata(); m != "" { log.Debugf("adding election metadata %s to queue", m) d.queue = append(d.queue, importItem{ + pid: p.GetProcessId(), uri: m, itemType: itemTypeElectionMetadata, })