From 5d8d652a3ce1f8a4855aae046c7ba7e378fc4e23 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 6 Jan 2023 10:45:41 +0100 Subject: [PATCH 1/7] Node will send node info even if there are no events in db (#2330) * Node will send node info even if there are no events in db * PR comments fixed --- src/commands/common/send-telemetry-command.js | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/src/commands/common/send-telemetry-command.js b/src/commands/common/send-telemetry-command.js index 7562b8e132..03fa2ff128 100644 --- a/src/commands/common/send-telemetry-command.js +++ b/src/commands/common/send-telemetry-command.js @@ -26,32 +26,30 @@ class SendTelemetryCommand extends Command { } try { const events = await this.telemetryInjectionService.getUnpublishedEvents(); - if (events && events.length > 0) { - const signalingMessage = { - nodeData: { - version: pjson.version, - identity: this.networkModuleManager.getPeerId().toB58String(), - hostname: this.config.hostname, - operational_wallet: this.blockchainModuleManager.getPublicKey(), - management_wallet: this.blockchainModuleManager.getManagementKey(), - triple_store: this.config.modules.tripleStore.defaultImplementation, - auto_update_enabled: this.config.modules.autoUpdater.enabled, - multiaddresses: this.networkModuleManager.getMultiaddrs(), - }, - events, - }; - const config = { - method: 'post', - url: this.config.telemetry.signalingServerUrl, - headers: { - 'Content-Type': 'application/json', - }, - data: JSON.stringify(signalingMessage), - }; - const response = await axios(config); - if (response.status === 200) { - await this.telemetryInjectionService.removePublishedEvents(events); - } + const signalingMessage = { + nodeData: { + version: pjson.version, + identity: this.networkModuleManager.getPeerId().toB58String(), + hostname: this.config.hostname, + operational_wallet: this.blockchainModuleManager.getPublicKey(), + management_wallet: this.blockchainModuleManager.getManagementKey(), + triple_store: this.config.modules.tripleStore.defaultImplementation, + auto_update_enabled: this.config.modules.autoUpdater.enabled, + multiaddresses: this.networkModuleManager.getMultiaddrs(), + }, + events: events || [], + }; + const config = { + method: 'post', + url: this.config.telemetry.signalingServerUrl, + headers: { + 'Content-Type': 'application/json', + }, + data: JSON.stringify(signalingMessage), + }; + const response = await axios(config); + if (response.status === 200 && events?.length > 0) { + await this.telemetryInjectionService.removePublishedEvents(events); } } catch (e) { await this.handleError(e); From 95844a3faff4029053d56dbc239b86a99843c269 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 19 Jan 2023 15:20:38 +0100 Subject: [PATCH 2/7] V6/update data layer (#2243) * fix local network setup * update data layer * update local store asset * fix merkle proof calculation * update triple store module according to new spec * fix bdd tests * minor bug fixes * add test data folders to gitignore * remove test data folders * remove test-data-bootstrap folder * decrease sleep time in test, remove code for asset update * add asset contract address validation, fix bdd tests * add local store http api route * minor fixes * add operation id in logs * add local-store to result controller * update local store to handle multiple store reqs * update localGet repositories * update query repository * declare services * triple store config migrations and local network setup updates (#2338) * Added creating namespaces for blazegraph, updated sparql endpoint for blazegraph * Added user configuration migration for triple store * add blazegraph to local network setup * add triple store repositories deletion * fix sparql endpoints * Updated fuseki database repository initialization * remove triple store repos from general config * Updated migration and repository deletion for fuseki * typo updated * add delete repositories functions in triple store module * fix fuseki urls * update config file names * remove config template from gitignore * refactor config file generation script * fix create and delete blazegraph repo * update repository exist functions * remove fuseki tdb * Updated triple store migration file * Template updated Co-authored-by: Djordje Kovacevic * workflows updated * BDD tests fixed * Triple store configuration migration updated after testing * update triple store config in installer * Fuseki migration updated * add node env selection to installer script * fix triple store config in installer * Updated delay for successfull publish * Updated node restart after migration * Event fetching interval for development and test udpated * fix fuseki service execution script * node stop after migration updated * remove empty triple store unit test file * fix blazegraph encoding err on assertion insertion * update publish and local store schema validation * update installer fuseki selection options * bump version Co-authored-by: Djordje Kovacevic --- .github/workflows/CHECK-lint.yml | 1 - .github/workflows/TEST-bdd.yml | 1 - .github/workflows/TEST-unit.yml | 1 - .gitignore | 8 +- .origintrail_noderc.tests | 26 -- config/config.json | 170 +++++------ frameDocuments/erc721.json | 9 - installer/data/fuseki.service | 2 +- installer/installer.sh | 68 ++++- ot-node.js | 27 +- package-lock.json | 4 +- package.json | 2 +- .../local-store/local-store-command.js | 67 +++++ .../v1.0.0/v1-0-0-handle-get-init-command.js | 9 +- .../v1-0-0-handle-get-request-command.js | 3 +- .../sender/get-latest-assertion-id-command.js | 62 ++--- .../protocols/get/sender/local-get-command.js | 43 ++- .../receiver/calculate-proofs-command.js | 11 +- .../v1-0-0-handle-store-init-command.js | 2 +- .../v1-0-0-handle-store-request-command.js | 9 +- .../publish/sender/local-store-command.js | 52 ---- .../sender/validate-assertion-command.js | 3 +- .../handle-search-assertions-init-command.js | 58 ---- ...andle-search-assertions-request-command.js | 65 ----- .../sender/local-search-assertions-command.js | 69 ----- .../sender/search-assertions-command.js | 53 ---- .../sender/search-assertions-init-command.js | 36 --- .../search-assertions-request-command.js | 70 ----- src/commands/query/query-command.js | 32 ++- src/constants/constants.js | 27 +- src/controllers/http-api/http-api-router.js | 13 +- .../local-store-http-api-controller.js | 46 +++ .../http-api/publish-http-api-controller.js | 5 +- .../http-api/query-http-api-controller.js | 38 +++ .../request-schema/local-store-schema.js | 22 ++ .../http-api/request-schema/publish-schema.js | 112 ++------ .../http-api/request-schema/search-schema.js | 20 -- .../http-api/result-http-api-controller.js | 6 +- .../http-api/search-http-api-controller.js | 156 ----------- src/controllers/rpc/search-rpc-controller.js | 36 --- ...iple-store-user-configuration-migration.js | 134 +++++++++ src/modules/base-module-manager.js | 34 +-- .../blockchain/implementation/web3-service.js | 25 +- src/modules/module-config-validation.js | 73 +++++ .../ot-blazegraph/ot-blazegraph.js | 101 ++++++- .../implementation/ot-fuseki/ot-fuseki.js | 67 ++++- .../implementation/ot-graphdb/ot-graphdb.js | 116 +++++--- .../implementation/ot-triple-store.js | 263 +++++++++--------- .../triple-store-module-manager.js | 112 ++++++-- src/service/data-service.js | 1 - src/service/get-service.js | 23 -- src/service/json-schema-service.js | 10 +- src/service/publish-service.js | 97 ------- src/service/query-service.js | 30 -- src/service/search-service.js | 32 --- src/service/triple-store-service.js | 215 ++++++++++++++ test/bdd/features/get-errors.feature | 8 +- test/bdd/features/publish-errors.feature | 2 +- test/bdd/features/release.feature | 6 +- test/bdd/steps/api/datasets/requests.json | 6 +- test/bdd/steps/api/publish.mjs | 13 +- test/bdd/steps/common.mjs | 113 ++++---- .../origintrail-test-bootstrap-config.json | 115 -------- .../config/origintrail-test-node-config.json | 109 -------- test/bdd/steps/hooks.mjs | 3 +- test/bdd/steps/lib/local-blockchain.mjs | 6 + test/unit/sparlql-query-service.test.js | 86 +++--- test/utilities/dkg-client-helper.mjs | 9 - test/utilities/steps-utils.mjs | 107 +++++-- .../.bootstrap_origintrail_noderc | 112 -------- .../.dh_origintrail_noderc | 88 ------ .../.origintrail_noderc_template.json | 179 ++++++++++++ .../generate-config-files.js | 220 +++++++++------ .../setup-macos-environment.sh | 26 +- 74 files changed, 1946 insertions(+), 2039 deletions(-) delete mode 100644 .origintrail_noderc.tests delete mode 100644 frameDocuments/erc721.json create mode 100644 src/commands/local-store/local-store-command.js delete mode 100644 src/commands/protocols/publish/sender/local-store-command.js delete mode 100644 src/commands/protocols/search/receiver/handle-search-assertions-init-command.js delete mode 100644 src/commands/protocols/search/receiver/handle-search-assertions-request-command.js delete mode 100644 src/commands/protocols/search/sender/local-search-assertions-command.js delete mode 100644 src/commands/protocols/search/sender/search-assertions-command.js delete mode 100644 src/commands/protocols/search/sender/search-assertions-init-command.js delete mode 100644 src/commands/protocols/search/sender/search-assertions-request-command.js create mode 100644 src/controllers/http-api/local-store-http-api-controller.js create mode 100644 src/controllers/http-api/query-http-api-controller.js create mode 100644 src/controllers/http-api/request-schema/local-store-schema.js delete mode 100644 src/controllers/http-api/request-schema/search-schema.js delete mode 100644 src/controllers/http-api/search-http-api-controller.js delete mode 100644 src/controllers/rpc/search-rpc-controller.js create mode 100644 src/migration/triple-store-user-configuration-migration.js create mode 100644 src/modules/module-config-validation.js delete mode 100644 src/service/query-service.js delete mode 100644 src/service/search-service.js create mode 100644 src/service/triple-store-service.js delete mode 100644 test/bdd/steps/config/origintrail-test-bootstrap-config.json delete mode 100644 test/bdd/steps/config/origintrail-test-node-config.json delete mode 100644 tools/local-network-setup/.bootstrap_origintrail_noderc delete mode 100644 tools/local-network-setup/.dh_origintrail_noderc create mode 100644 tools/local-network-setup/.origintrail_noderc_template.json diff --git a/.github/workflows/CHECK-lint.yml b/.github/workflows/CHECK-lint.yml index b2408d2161..a8fca282c7 100644 --- a/.github/workflows/CHECK-lint.yml +++ b/.github/workflows/CHECK-lint.yml @@ -21,7 +21,6 @@ jobs: node-version: ${{ matrix.node-version }} - run: npm install - run: mkdir -p $ARTIFACTS_DIR - - run: cp .origintrail_noderc.tests .origintrail_noderc - run: sudo chmod -R 777 $ARTIFACTS_DIR - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR diff --git a/.github/workflows/TEST-bdd.yml b/.github/workflows/TEST-bdd.yml index c940b0ad13..9ffe70e64b 100644 --- a/.github/workflows/TEST-bdd.yml +++ b/.github/workflows/TEST-bdd.yml @@ -38,7 +38,6 @@ jobs: node-version: ${{ matrix.node-version }} - run: npm install - run: mkdir -p $ARTIFACTS_DIR - - run: cp .origintrail_noderc.tests .origintrail_noderc - run: sudo chmod -R 777 $ARTIFACTS_DIR - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR diff --git a/.github/workflows/TEST-unit.yml b/.github/workflows/TEST-unit.yml index 401d05fca7..9b437e88e4 100644 --- a/.github/workflows/TEST-unit.yml +++ b/.github/workflows/TEST-unit.yml @@ -38,7 +38,6 @@ jobs: node-version: ${{ matrix.node-version }} - run: npm install - run: mkdir -p $ARTIFACTS_DIR - - run: cp .origintrail_noderc.tests .origintrail_noderc - run: sudo chmod -R 777 $ARTIFACTS_DIR - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR diff --git a/.gitignore b/.gitignore index 493ced6822..be72117326 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ yarn-error.log* lerna-debug.log* .idea .origintrail_noderc -.*_origintrail_noderc +.*_origintrail_noderc.json # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json @@ -107,5 +107,11 @@ dist .DS_Store +# Test data folders +test-data* + +# Data folders data* + +# VS code files .vscode/launch.json diff --git a/.origintrail_noderc.tests b/.origintrail_noderc.tests deleted file mode 100644 index 11b978aa9d..0000000000 --- a/.origintrail_noderc.tests +++ /dev/null @@ -1,26 +0,0 @@ -{ - "blockchain":[ - { - "blockchainTitle": "Polygon", - "networkId": "polygon::testnet", - "rpcEndpoints": ["https://rpc-mumbai.maticvigil.com/"], - "publicKey": "...", - "privateKey": "..." - } - ], - "graphDatabase": { - "name": "node0", - "username": "admin", - "password": "", - "implementation": "GraphDB", - "url": "http://localhost:7200/" - }, - "logLevel": "trace", - "rpcPort": 8900, - "network": { - }, - "ipWhitelist": [ - "::1", - "127.0.0.1" - ] -} diff --git a/config/config.json b/config/config.json index a734b3208a..c91d8884fa 100644 --- a/config/config.json +++ b/config/config.json @@ -5,6 +5,7 @@ "enabled": false, "implementation": { "ot-auto-updater": { + "enabled": false, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v6/develop" @@ -16,6 +17,7 @@ "enabled": true, "implementation": { "express-http-client": { + "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, @@ -34,6 +36,7 @@ "enabled": true, "implementation": { "libp2p-service": { + "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { @@ -62,6 +65,7 @@ "enabled": true, "implementation": { "sequelize-repository": { + "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", @@ -77,34 +81,21 @@ }, "tripleStore": { "enabled": true, - "defaultImplementation": "ot-graphdb", "implementation": { "ot-blazegraph": { + "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", - "config": { - "url": "http://localhost:9999/blazegraph", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} }, "ot-fuseki": { + "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", - "config": { - "url": "http://localhost:3030", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} }, "ot-graphdb": { + "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", - "config": { - "url": "http://localhost:7200", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} } } }, @@ -112,6 +103,7 @@ "enabled": true, "implementation": { "merkle-validation": { + "enabled": true, "package": "./validation/implementation/merkle-validation.js", "config": {} } @@ -121,6 +113,7 @@ "enabled": true, "implementation": { "ganache": { + "enabled": true, "package": "./blockchain/implementation/ganache/ganache-service.js", "config": { "blockchainTitle": "ganache", @@ -133,6 +126,7 @@ } }, "otp": { + "enabled": false, "package": "./blockchain/implementation/ot-parachain/ot-parachain-service.js", "config": { "networkId": "otp::testnet", @@ -143,6 +137,7 @@ } }, "polygon": { + "enabled": false, "package": "./blockchain/implementation/polygon/polygon-service.js", "config": { "networkId": "polygon::testnet", @@ -159,6 +154,7 @@ } }, "rinkeby": { + "enabled": false, "package": "./blockchain/implementation/polygon/eth-service.js", "config": { "networkId": "eth::rinkeby", @@ -194,6 +190,7 @@ "enabled": false, "implementation": { "ot-auto-updater": { + "enabled": false, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v6/develop" @@ -205,8 +202,17 @@ "enabled": true, "implementation": { "express-http-client": { + "enabled": true, "package": "./http-client/implementation/express-http-client.js", - "config": {} + "config": { + "useSsl": false, + "sslKeyPath": "/root/certs/privkey.pem", + "sslCertificatePath": "/root/certs/fullchain.pem", + "rateLimiter": { + "timeWindowSeconds": 60, + "maxRequests": 10 + } + } } } }, @@ -214,6 +220,7 @@ "enabled": true, "implementation": { "libp2p-service": { + "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { @@ -239,15 +246,26 @@ } } }, + "validation": { + "enabled": true, + "implementation": { + "enabled": true, + "merkle-validation": { + "package": "./validation/implementation/merkle-validation.js", + "config": {} + } + } + }, "repository": { "enabled": true, "implementation": { "sequelize-repository": { + "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", "user": "root", - "password": "password", + "password": "", "port": "3306", "host": "localhost", "dialect": "mysql", @@ -258,9 +276,9 @@ }, "blockchain": { "enabled": true, - "defaultImplementation": "ganache", "implementation": { "ganache": { + "enabled": true, "package": "./blockchain/implementation/ganache/ganache-service.js", "config": { "blockchainTitle": "ganache", @@ -275,34 +293,11 @@ }, "tripleStore": { "enabled": true, - "defaultImplementation": "ot-graphdb", "implementation": { - "ot-blazegraph": { - "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", - "config": { - "url": "http://localhost:9999/blazegraph", - "repository": "node0", - "username": "admin", - "password": "" - } - }, - "ot-fuseki": { - "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", - "config": { - "url": "http://localhost:3030", - "repository": "node0", - "username": "admin", - "password": "" - } - }, "ot-graphdb": { + "enabled": true, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", - "config": { - "url": "http://localhost:7200", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} } } } @@ -329,6 +324,7 @@ "enabled": true, "implementation": { "ot-auto-updater": { + "enabled": true, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v6/release/testnet" @@ -340,6 +336,7 @@ "enabled": true, "implementation": { "libp2p-service": { + "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { @@ -370,6 +367,7 @@ "enabled": true, "implementation": { "express-http-client": { + "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, @@ -388,6 +386,7 @@ "enabled": true, "implementation": { "sequelize-repository": { + "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", @@ -403,9 +402,9 @@ }, "blockchain": { "enabled": true, - "defaultImplementation": "otp", "implementation": { "otp": { + "enabled": true, "package": "./blockchain/implementation/ot-parachain/ot-parachain-service.js", "config": { "networkId": "parachain::testnet", @@ -422,6 +421,7 @@ "enabled": true, "implementation": { "merkle-validation": { + "enabled": true, "package": "./validation/implementation/merkle-validation.js", "config": {} } @@ -429,34 +429,21 @@ }, "tripleStore": { "enabled": true, - "defaultImplementation": "ot-blazegraph", "implementation": { "ot-blazegraph": { + "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", - "config": { - "url": "http://localhost:9999/blazegraph", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} }, "ot-fuseki": { + "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", - "config": { - "url": "http://localhost:3030", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} }, "ot-graphdb": { + "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", - "config": { - "url": "http://localhost:7200", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} } } } @@ -483,6 +470,7 @@ "enabled": true, "implementation": { "ot-auto-updater": { + "enabled": true, "package": "./auto-updater/implementation/ot-auto-updater.js", "config": { "branch": "v6/release/mainnet" @@ -494,6 +482,7 @@ "enabled": true, "implementation": { "libp2p-service": { + "enabled": true, "package": "./network/implementation/libp2p-service.js", "config": { "dht": { @@ -524,6 +513,7 @@ "enabled": true, "implementation": { "express-http-client": { + "enabled": true, "package": "./http-client/implementation/express-http-client.js", "config": { "useSsl": false, @@ -542,6 +532,7 @@ "enabled": true, "implementation": { "sequelize-repository": { + "enabled": true, "package": "./repository/implementation/sequelize/sequelize-repository.js", "config": { "database": "operationaldb", @@ -560,6 +551,7 @@ "defaultImplementation": "otp", "implementation": { "otp": { + "enabled": true, "package": "./blockchain/implementation/ot-parachain/ot-parachain-service.js", "config": { "networkId": "otp::mainnet", @@ -573,45 +565,33 @@ } } }, - "validation": { - "enabled": true, - "implementation": { - "merkle-validation": { - "package": "./validation/implementation/merkle-validation.js", - "config": {} - } - } - }, "tripleStore": { "enabled": true, - "defaultImplementation": "ot-blazegraph", "implementation": { "ot-blazegraph": { + "enabled": false, "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", - "config": { - "url": "http://localhost:9999/blazegraph", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} }, "ot-fuseki": { + "enabled": false, "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", - "config": { - "url": "http://localhost:3030", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} }, "ot-graphdb": { + "enabled": false, "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", - "config": { - "url": "http://localhost:7200", - "repository": "node0", - "username": "admin", - "password": "" - } + "config": {} + } + } + }, + "validation": { + "enabled": true, + "implementation": { + "merkle-validation": { + "enabled": true, + "package": "./validation/implementation/merkle-validation.js", + "config": {} } } } diff --git a/frameDocuments/erc721.json b/frameDocuments/erc721.json deleted file mode 100644 index b842c65bac..0000000000 --- a/frameDocuments/erc721.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "context": { - "@context": "https://www.schema.org/" - }, - "frame": { - "@context": "https://www.schema.org/", - "@type": "ERC721" - } -} diff --git a/installer/data/fuseki.service b/installer/data/fuseki.service index efdd6251f1..48b4da31ca 100644 --- a/installer/data/fuseki.service +++ b/installer/data/fuseki.service @@ -10,7 +10,7 @@ Environment=JVM_ARGS=-Xmx4G Type=simple User=root WorkingDirectory=/root/ot-node/fuseki -ExecStart=/usr/bin/java -jar /root/ot-node/fuseki/fuseki-server.jar --update --set tdb:unionDefaultGraph=true --loc /root/ot-node/fuseki/tdb /node0 +ExecStart=/usr/bin/java -jar /root/ot-node/fuseki/fuseki-server.jar --update --set tdb:unionDefaultGraph=true Restart=on-failure [Install] diff --git a/installer/installer.sh b/installer/installer.sh index d750adcb5d..ba6a78771b 100755 --- a/installer/installer.sh +++ b/installer/installer.sh @@ -104,7 +104,6 @@ install_fuseki() { perform_step unzip $FUSEKI_VER.zip "Unzipping Fuseki" perform_step rm /root/$FUSEKI_VER.zip "Removing Fuseki zip file" perform_step mkdir /root/ot-node/fuseki "Making /root/ot-node/fuseki directory" - perform_step mkdir /root/ot-node/fuseki/tdb "Making /root/ot-node/fuseki/tdb directory" perform_step cp /root/$FUSEKI_VER/fuseki-server.jar /root/ot-node/fuseki/ "Copying Fuseki files to $OTNODE_DIR/fuseki/ 1/2" perform_step cp -r /root/$FUSEKI_VER/webapp/ /root/ot-node/fuseki/ "Copying Fuseki files to $OTNODE_DIR/fuseki/ 1/2" perform_step rm -r /root/$FUSEKI_VER "Removing the remaining /root/$FUSEKI_VER directory" @@ -206,8 +205,17 @@ install_sql() { } install_node() { + # Change directory to ot-node/current + cd $OTNODE_DIR - CONFIG_DIR=$OTNODE_DIR/.. + #request node env + read -p "Please select node environment: (Default: Mainnet) [T]estnet [M]ainnet [E]xit " choice + case "$choice" in + [tT]* ) nodeEnv="testnet";; + [eE]* ) text_color $RED"Installer stopped by user"; exit;; + * ) nodeEnv="mainnet";; + esac + echo "NODE_ENV=$nodeEnv" >> $OTNODE_DIR/.env #blockchains=("otp" "polygon") #for ((i = 0; i < ${#blockchains[@]}; ++i)); @@ -248,18 +256,56 @@ install_node() { # * ) ((--i));echo "Please make a valid choice and try again.";; #esac #done - - # Change directory to ot-node/current - cd $OTNODE_DIR perform_step npm ci --omit=dev --ignore-scripts "Executing npm install" - echo "NODE_ENV=mainnet" >> $OTNODE_DIR/.env - + CONFIG_DIR=$OTNODE_DIR/.. perform_step touch $CONFIG_DIR/.origintrail_noderc "Configuring node config file" - perform_step $(jq --null-input --arg tripleStore "$tripleStore" '{"logLevel": "trace", "auth": {"ipWhitelist": ["::1", "127.0.0.1"]}, "modules": {"tripleStore":{"defaultImplementation": $tripleStore}}}' > $CONFIG_DIR/.origintrail_noderc) "Adding tripleStore $tripleStore to node config file" + perform_step $(jq --null-input --arg tripleStore "$tripleStore" '{"logLevel": "trace", "auth": {"ipWhitelist": ["::1", "127.0.0.1"]}}' > $CONFIG_DIR/.origintrail_noderc) "Adding loglevel and auth values to node config file" + + perform_step $(jq --arg tripleStore "$tripleStore" --arg tripleStoreUrl "$tripleStoreUrl" '.modules.tripleStore.implementation[$tripleStore] |= + { + "enabled": "true", + "config": { + "repositories": { + "privateCurrent": { + "url": $tripleStoreUrl, + "name": "private-current", + "username": "admin", + "password": "" + }, + "privateHistory": { + "url": $tripleStoreUrl, + "name": "private-history", + "username": "admin", + "password": "" + }, + "publicCurrent": { + "url": $tripleStoreUrl, + "name": "public-current", + "username": "admin", + "password": "" + }, + "publicHistory": { + "url": $tripleStoreUrl, + "name": "public-history", + "username": "admin", + "password": "" + } + } + } + } + .' $CONFIG_DIR/.origintrail_noderc > $CONFIG_DIR/origintrail_noderc_tmp) "Adding node wallets to node config file 1/2" - perform_step $(jq --arg blockchain "otp" --arg evmOperationalWallet "$EVM_OPERATIONAL_WALLET" --arg evmOperationalWalletPrivateKey "$EVM_OPERATIONAL_PRIVATE_KEY" --arg evmManagementWallet "$EVM_MANAGEMENT_WALLET" --arg evmManagementWallet "$SHARES_TOKEN_NAME" --arg evmManagementWallet "$SHARES_TOKEN_SYMBOL" --arg sharesTokenName "$SHARES_TOKEN_NAME" --arg sharesTokenSymbol "$SHARES_TOKEN_SYMBOL" '.modules.blockchain.implementation[$blockchain].config |= { "evmOperationalWalletPublicKey": $evmOperationalWallet, "evmOperationalWalletPrivateKey": $evmOperationalWalletPrivateKey, "evmManagementWalletPublicKey": $evmManagementWallet, "sharesTokenName": $sharesTokenName, "sharesTokenSymbol": $sharesTokenSymbol} + .' $CONFIG_DIR/.origintrail_noderc > $CONFIG_DIR/origintrail_noderc_tmp) "Adding node wallets to node config file 1/2" + perform_step mv $CONFIG_DIR/origintrail_noderc_tmp $CONFIG_DIR/.origintrail_noderc "Adding node wallets to node config file 2/2" + + perform_step $(jq --arg blockchain "otp" --arg evmOperationalWallet "$EVM_OPERATIONAL_WALLET" --arg evmOperationalWalletPrivateKey "$EVM_OPERATIONAL_PRIVATE_KEY" --arg evmManagementWallet "$EVM_MANAGEMENT_WALLET" --arg evmManagementWallet "$SHARES_TOKEN_NAME" --arg evmManagementWallet "$SHARES_TOKEN_SYMBOL" --arg sharesTokenName "$SHARES_TOKEN_NAME" --arg sharesTokenSymbol "$SHARES_TOKEN_SYMBOL" '.modules.blockchain.implementation[$blockchain].config |= + { + "evmOperationalWalletPublicKey": $evmOperationalWallet, + "evmOperationalWalletPrivateKey": $evmOperationalWalletPrivateKey, + "evmManagementWalletPublicKey": $evmManagementWallet, + "sharesTokenName": $sharesTokenName, + "sharesTokenSymbol": $sharesTokenSymbol + } + .' $CONFIG_DIR/.origintrail_noderc > $CONFIG_DIR/origintrail_noderc_tmp) "Adding node wallets to node config file 1/2" perform_step mv $CONFIG_DIR/origintrail_noderc_tmp $CONFIG_DIR/.origintrail_noderc "Adding node wallets to node config file 2/2" @@ -306,9 +352,9 @@ header_color $BGREEN"Installing Triplestore (Graph Database)..." read -p "Please select the database you would like to use: (Default: Blazegraph) [1]Blazegraph [2]Fuseki [E]xit: " choice case "$choice" in - [2fF] ) text_color $GREEN"Fuseki selected. Proceeding with installation."; tripleStore=ot-fuseki;; + [2] ) text_color $GREEN"Fuseki selected. Proceeding with installation."; tripleStore=ot-fuseki; tripleStoreUrl="http://localhost:3030";; [Ee] ) text_color $RED"Installer stopped by user"; exit;; - * ) text_color $GREEN"Blazegraph selected. Proceeding with installation."; tripleStore=ot-blazegraph;; + * ) text_color $GREEN"Blazegraph selected. Proceeding with installation."; tripleStore=ot-blazegraph; tripleStoreUrl="http://localhost:9999";; esac if [[ $tripleStore = "ot-fuseki" ]]; then diff --git a/ot-node.js b/ot-node.js index ba1b9167a2..8e7eb4e915 100644 --- a/ot-node.js +++ b/ot-node.js @@ -10,6 +10,7 @@ import FileService from './src/service/file-service.js'; import OtnodeUpdateCommand from './src/commands/common/otnode-update-command.js'; import OtAutoUpdater from './src/modules/auto-updater/implementation/ot-auto-updater.js'; import PullBlockchainShardingTableMigration from './src/migration/pull-sharding-table-migration.js'; +import TripleStoreUserConfigurationMigration from './src/migration/triple-store-user-configuration-migration.js'; const require = createRequire(import.meta.url); const pjson = require('./package.json'); @@ -27,7 +28,7 @@ class OTNode { async start() { await this.checkForUpdate(); await this.removeUpdateFile(); - + await this.executeTripleStoreUserConfigurationMigration(); this.logger.info(' ██████╗ ████████╗███╗ ██╗ ██████╗ ██████╗ ███████╗'); this.logger.info('██╔═══██╗╚══██╔══╝████╗ ██║██╔═══██╗██╔══██╗██╔════╝'); this.logger.info('██║ ██║ ██║ ██╔██╗ ██║██║ ██║██║ ██║█████╗'); @@ -71,7 +72,7 @@ class OTNode { } initializeLogger() { - this.logger = new Logger(this.config.logLevel, this.config.telemetry.enabled); + this.logger = new Logger(this.config.logLevel); } initializeFileService() { @@ -222,6 +223,21 @@ class OTNode { } } + async executeTripleStoreUserConfigurationMigration() { + if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test') return; + + const migration = new TripleStoreUserConfigurationMigration( + 'tripleStoreUserConfigurationMigration', + this.logger, + this.config, + ); + if (!(await migration.migrationAlreadyExecuted())) { + await migration.migrate(); + this.logger.info('Node will now restart!'); + this.stop(1); + } + } + async executePullShardingTableMigration() { if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test') return; @@ -301,7 +317,10 @@ class OTNode { ); let working = false; - + let eventFetchInterval = 10 * 1000; + if (process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test') { + eventFetchInterval = 2; + } setInterval(async () => { if (!working) { try { @@ -336,7 +355,7 @@ class OTNode { working = false; } } - }, 10 * 1000); + }, eventFetchInterval); } async initializeTelemetryInjectionService() { diff --git a/package-lock.json b/package-lock.json index e7f3bce8ad..6b8897dbf8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.2", + "version": "6.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.2", + "version": "6.0.3", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 24b59e7129..dfa506fb03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.2", + "version": "6.0.3", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/local-store/local-store-command.js b/src/commands/local-store/local-store-command.js new file mode 100644 index 0000000000..536cd45632 --- /dev/null +++ b/src/commands/local-store/local-store-command.js @@ -0,0 +1,67 @@ +import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../constants/constants.js'; +import Command from '../command.js'; + +class LocalStoreCommand extends Command { + constructor(ctx) { + super(ctx); + this.tripleStoreService = ctx.tripleStoreService; + + this.errorType = ERROR_TYPE.LOCAL_STORE.LOCAL_STORE_ERROR; + } + + async execute(command) { + const { operationId } = command.data; + + try { + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.LOCAL_STORE.LOCAL_STORE_START, + ); + + const assertions = await this.operationIdService.getCachedOperationIdData(operationId); + + await Promise.all( + assertions.map(({ assertionId, assertion }) => + this.tripleStoreService.localStoreAssertion( + assertionId, + assertion, + operationId, + ), + ), + ); + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.LOCAL_STORE.LOCAL_STORE_END, + ); + + await this.operationIdService.cacheOperationIdData(operationId, {}); + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.COMPLETED, + ); + } catch (e) { + await this.handleError(operationId, e.message, this.errorType, true); + } + + return Command.empty(); + } + + /** + * Builds default localStoreCommand + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'localStoreCommand', + delay: 0, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default LocalStoreCommand; diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js index 49face2898..283f164e8c 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js @@ -1,15 +1,15 @@ import HandleProtocolMessageCommand from '../../../common/handle-protocol-message-command.js'; - import { ERROR_TYPE, NETWORK_MESSAGE_TYPES, OPERATION_ID_STATUS, + TRIPLE_STORE_REPOSITORIES, } from '../../../../../constants/constants.js'; class HandleGetInitCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; + this.tripleStoreService = ctx.tripleStoreService; this.operationService = ctx.getService; this.errorType = ERROR_TYPE.GET.GET_INIT_REMOTE_ERROR; @@ -22,7 +22,10 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { OPERATION_ID_STATUS.GET.ASSERTION_EXISTS_LOCAL_START, ); - const assertionExists = await this.tripleStoreModuleManager.assertionExists(assertionId); + const assertionExists = await this.tripleStoreService.assertionExists( + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + ); await this.operationIdService.updateOperationIdStatus( operationId, diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js index 6e7f9bbfe6..83b1b8c4fb 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js @@ -10,6 +10,7 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { constructor(ctx) { super(ctx); this.operationService = ctx.getService; + this.tripleStoreService = ctx.tripleStoreService; this.errorType = ERROR_TYPE.GET.GET_REQUEST_REMOTE_ERROR; } @@ -21,7 +22,7 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { OPERATION_ID_STATUS.GET.GET_REMOTE_START, ); - const nquads = await this.operationService.localGet(assertionId, operationId); + const nquads = await this.tripleStoreService.localGet(assertionId, operationId); await this.operationIdService.updateOperationIdStatus( operationId, diff --git a/src/commands/protocols/get/sender/get-latest-assertion-id-command.js b/src/commands/protocols/get/sender/get-latest-assertion-id-command.js index 2645dd9e0d..109d691770 100644 --- a/src/commands/protocols/get/sender/get-latest-assertion-id-command.js +++ b/src/commands/protocols/get/sender/get-latest-assertion-id-command.js @@ -17,47 +17,45 @@ class GetLatestAssertionIdCommand extends Command { async execute(command) { const { id, operationId } = command.data; - // id options : - // did:otp:0x174714134abcd13431413413/987654321/41eaa20f35f709d9c22281f46d895b2f7a83c54587e4339456e0d9f4e5bd9b8f - // did:otp:0x174714134abcd13431413413/987654321/latest - const commandData = {}; if (!this.ualService.isUAL(id)) { this.handleError(operationId, `Requested id is not a UAL`, this.errorType, true); + + return Command.empty(); + } + + const { + blockchain, + contract, + tokenId, + assertionId: ualAssertionId, + } = this.ualService.resolveUAL(id); + commandData.blockchain = blockchain; + commandData.tokenId = tokenId; + commandData.contract = contract; + + if (ualAssertionId && ualAssertionId !== 'latest') { + commandData.assertionId = ualAssertionId; } else { - const { + this.logger.debug( + `Searching for assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, + ); + const blockchainAssertionId = await this.blockchainModuleManager.getLatestAssertionId( blockchain, contract, tokenId, - assertionId: ualAssertionId, - } = this.ualService.resolveUAL(id); - commandData.blockchain = blockchain; - commandData.tokenId = tokenId; - commandData.contract = contract; - - if (ualAssertionId && ualAssertionId !== 'latest') { - commandData.assertionId = ualAssertionId; - } else { - this.logger.debug( - `Searching for assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, + ); + if (!blockchainAssertionId) { + this.handleError( + operationId, + `Unable to find assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, + this.errorType, + true, ); - const blockchainAssertionId = - await this.blockchainModuleManager.getLatestAssertionId( - blockchain, - contract, - tokenId, - ); - if (!blockchainAssertionId) { - this.handleError( - operationId, - `Unable to find assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, - this.errorType, - true, - ); - return Command.empty(); - } - commandData.assertionId = blockchainAssertionId; + + return Command.empty(); } + commandData.assertionId = blockchainAssertionId; } return this.continueSequence({ ...command.data, ...commandData }, command.sequence); diff --git a/src/commands/protocols/get/sender/local-get-command.js b/src/commands/protocols/get/sender/local-get-command.js index 2efb8fb302..329f2a1780 100644 --- a/src/commands/protocols/get/sender/local-get-command.js +++ b/src/commands/protocols/get/sender/local-get-command.js @@ -6,8 +6,7 @@ class LocalGetCommand extends Command { super(ctx); this.config = ctx.config; this.operationIdService = ctx.operationIdService; - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; - this.getService = ctx.getService; + this.tripleStoreService = ctx.tripleStoreService; this.errorType = ERROR_TYPE.GET.GET_LOCAL_ERROR; } @@ -22,29 +21,27 @@ class LocalGetCommand extends Command { operationId, OPERATION_ID_STATUS.GET.GET_LOCAL_START, ); - const assertionExists = await this.tripleStoreModuleManager.assertionExists(assertionId); - if (assertionExists) { - const assertion = await this.getService.localGet(assertionId, operationId); - if (assertion.length) { - await this.operationIdService.cacheOperationIdData(operationId, { - assertion, - }); - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.GET.GET_LOCAL_END, - ); - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.GET.GET_END, - ); - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.COMPLETED, - ); + const assertion = await this.tripleStoreService.localGet(assertionId, operationId, true); - return Command.empty(); - } + if (assertion.length) { + await this.operationIdService.cacheOperationIdData(operationId, { + assertion, + }); + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.GET.GET_LOCAL_END, + ); + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.GET.GET_END, + ); + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.COMPLETED, + ); + + return Command.empty(); } await this.operationIdService.updateOperationIdStatus( diff --git a/src/commands/protocols/publish/receiver/calculate-proofs-command.js b/src/commands/protocols/publish/receiver/calculate-proofs-command.js index 40b97ed88d..bb33fd3048 100644 --- a/src/commands/protocols/publish/receiver/calculate-proofs-command.js +++ b/src/commands/protocols/publish/receiver/calculate-proofs-command.js @@ -2,6 +2,7 @@ import EpochCommand from '../../common/epoch-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, + TRIPLE_STORE_REPOSITORIES, COMMAND_RETRIES, } from '../../../../constants/constants.js'; @@ -11,7 +12,7 @@ class CalculateProofsCommand extends EpochCommand { this.commandExecutor = ctx.commandExecutor; this.validationModuleManager = ctx.validationModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; + this.tripleStoreService = ctx.tripleStoreService; this.operationIdService = ctx.operationIdService; this.dataService = ctx.dataService; this.errorType = ERROR_TYPE.CALCULATE_PROOFS_ERROR; @@ -68,9 +69,11 @@ class CalculateProofsCommand extends EpochCommand { epoch, ); - let nquads = await this.tripleStoreModuleManager.get(assertionId); - - nquads = await this.dataService.toNQuads(nquads, 'application/n-quads'); + const nquads = await this.tripleStoreService.localGet( + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + operationId, + ); const { leaf, proof } = this.validationModuleManager.getMerkleProof( nquads, diff --git a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js index 8d1b474d87..7ce6f38a3f 100644 --- a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js +++ b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-init-command.js @@ -97,7 +97,7 @@ class HandleStoreInitCommand extends HandleProtocolMessageCommand { } async validateAssertionId(blockchain, contract, tokenId, assertionId, ual) { - const blockchainAssertionId = await this.publishService.getLatestAssertionId( + const blockchainAssertionId = await this.blockchainModuleManager.getLatestAssertionId( blockchain, contract, tokenId, diff --git a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js index b95ac72861..bdbc899301 100644 --- a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js +++ b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js @@ -5,6 +5,7 @@ import { OPERATION_ID_STATUS, ERROR_TYPE, AGREEMENT_STATUS, + TRIPLE_STORE_REPOSITORIES, } from '../../../../../constants/constants.js'; class HandleStoreRequestCommand extends HandleProtocolMessageCommand { @@ -15,7 +16,7 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { this.commandExecutor = ctx.commandExecutor; this.repositoryModuleManager = ctx.repositoryModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; + this.tripleStoreService = ctx.tripleStoreService; this.ualService = ctx.ualService; this.errorType = ERROR_TYPE.PUBLISH.PUBLISH_LOCAL_STORE_REMOTE_ERROR; @@ -52,7 +53,8 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { ); const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); - const assetExists = await this.tripleStoreModuleManager.assetExists( + const assetExists = await this.tripleStoreService.assetExists( + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, ual, blockchain, contract, @@ -62,8 +64,9 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { const agreementEndTime = agreementData.startTime + agreementData.epochsNumber * agreementData.epochsLength; - await this.operationService.localStoreAsset( + await this.tripleStoreService.localStoreAsset( assertionId, + assertion, blockchain, contract, tokenId, diff --git a/src/commands/protocols/publish/sender/local-store-command.js b/src/commands/protocols/publish/sender/local-store-command.js deleted file mode 100644 index 15af0dab71..0000000000 --- a/src/commands/protocols/publish/sender/local-store-command.js +++ /dev/null @@ -1,52 +0,0 @@ -import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../../constants/constants.js'; -import Command from '../../../command.js'; - -class LocalStoreCommand extends Command { - constructor(ctx) { - super(ctx); - this.operationService = ctx.publishService; - - this.errorType = ERROR_TYPE.PUBLISH.PUBLISH_LOCAL_STORE_ERROR; - } - - async execute(command) { - const { operationId, assertionId } = command.data; - - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_START, - ); - - await this.operationService.localStoreAsset( - assertionId, - command.data.blockchain, - command.data.contract, - command.data.tokenId, - operationId, - ); - - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_END, - ); - - return this.continueSequence(command.data, command.sequence); - } - - /** - * Builds default localStoreCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'localStoreCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default LocalStoreCommand; diff --git a/src/commands/protocols/publish/sender/validate-assertion-command.js b/src/commands/protocols/publish/sender/validate-assertion-command.js index 0c1d91091d..ca37e0249a 100644 --- a/src/commands/protocols/publish/sender/validate-assertion-command.js +++ b/src/commands/protocols/publish/sender/validate-assertion-command.js @@ -4,6 +4,7 @@ import { ERROR_TYPE, OPERATION_ID_STATUS } from '../../../../constants/constants class ValidateAssertionCommand extends Command { constructor(ctx) { super(ctx); + this.blockchainModuleManager = ctx.blockchainModuleManager; this.operationService = ctx.publishService; this.ualService = ctx.ualService; @@ -25,7 +26,7 @@ class ValidateAssertionCommand extends Command { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.info(`Validating assertion with ual: ${ual}`); - const blockchainAssertionId = await this.operationService.getLatestAssertionId( + const blockchainAssertionId = await this.blockchainModuleManager.getLatestAssertionId( blockchain, contract, tokenId, diff --git a/src/commands/protocols/search/receiver/handle-search-assertions-init-command.js b/src/commands/protocols/search/receiver/handle-search-assertions-init-command.js deleted file mode 100644 index ecc925a79d..0000000000 --- a/src/commands/protocols/search/receiver/handle-search-assertions-init-command.js +++ /dev/null @@ -1,58 +0,0 @@ -import Command from '../../../command.js'; -import { - ERROR_TYPE, - NETWORK_MESSAGE_TYPES, - NETWORK_PROTOCOLS, -} from '../../../../constants/constants.js'; - -class HandleSearchAssertionsInitCommand extends Command { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.config = ctx.config; - this.commandExecutor = ctx.commandExecutor; - this.networkModuleManager = ctx.networkModuleManager; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute(command) { - const { remotePeerId, operationId } = command.data; - - const messageType = NETWORK_MESSAGE_TYPES.RESPONSES.ACK; - const messageData = {}; - await this.networkModuleManager.sendMessageResponse( - NETWORK_PROTOCOLS.SEARCH_ASSERTIONS, - remotePeerId, - messageType, - operationId, - messageData, - ); - - return this.continueSequence(command.data, command.sequence); - } - - handleError(operationId, error, msg) { - this.logger.error(msg); - } - - /** - * Builds default handleSearchAssertionsInitCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'handleSearchAssertionsInitCommand', - delay: 0, - transactional: false, - errorType: ERROR_TYPE.HANDLE_SEARCH_ASSERTIONS_INIT_ERROR, - }; - Object.assign(command, map); - return command; - } -} - -export default HandleSearchAssertionsInitCommand; diff --git a/src/commands/protocols/search/receiver/handle-search-assertions-request-command.js b/src/commands/protocols/search/receiver/handle-search-assertions-request-command.js deleted file mode 100644 index 7ea0af1e7f..0000000000 --- a/src/commands/protocols/search/receiver/handle-search-assertions-request-command.js +++ /dev/null @@ -1,65 +0,0 @@ -import Command from '../../../command.js'; -import { - ERROR_TYPE, - NETWORK_MESSAGE_TYPES, - NETWORK_PROTOCOLS, -} from '../../../../constants/constants.js'; - -class HandleSearchAssertionsRequestCommand extends Command { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.config = ctx.config; - this.networkModuleManager = ctx.networkModuleManager; - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute(command) { - const { message, remotePeerId, operationId } = command.data; - - const localQuery = true; - const data = await this.tripleStoreModuleManager.findAssertionsByKeyword( - message.data.query, - message.data.options, - localQuery, - ); - - const messageType = NETWORK_MESSAGE_TYPES.RESPONSES.ACK; - const messageData = data.map((assertion) => assertion.assertionId); - await this.networkModuleManager.sendMessageResponse( - NETWORK_PROTOCOLS.SEARCH_ASSERTIONS, - remotePeerId, - messageType, - operationId, - messageData, - ); - - return Command.empty(); - } - - handleError(operationId, error, msg) { - this.logger.error(msg); - } - - /** - * Builds default handleSearchAssertionsRequestCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'handleSearchAssertionsRequestCommand', - delay: 0, - transactional: false, - errorType: ERROR_TYPE.HANDLE_SEARCH_ASSERTIONS_REQUEST_ERROR, - }; - Object.assign(command, map); - return command; - } -} - -export default HandleSearchAssertionsRequestCommand; diff --git a/src/commands/protocols/search/sender/local-search-assertions-command.js b/src/commands/protocols/search/sender/local-search-assertions-command.js deleted file mode 100644 index 403378d0f1..0000000000 --- a/src/commands/protocols/search/sender/local-search-assertions-command.js +++ /dev/null @@ -1,69 +0,0 @@ -import Command from '../../../command.js'; -import { OPERATION_ID_STATUS } from '../../../../constants/constants.js'; - -class LocalSearchAssertionsCommand extends Command { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.config = ctx.config; - this.networkModuleManager = ctx.networkModuleManager; - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; - this.operationIdService = ctx.operationIdService; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute(command) { - const { operationId, query, options } = command.data; - - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.SEARCH_ASSERTIONS.SEARCHING_ASSERTIONS, - ); - - try { - const localQuery = true; - const response = await this.tripleStoreModuleManager.findAssertionsByKeyword( - query, - options, - localQuery, - ); - - const data = {}; - data.assertions = response.map((assertion) => assertion.assertionId); - - await this.operationIdService.cacheOperationIdData(operationId, data); - } catch (e) { - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.FAILED, - e.message, - ); - } - - return this.continueSequence(command.data, command.sequence); - } - - handleError(operationId, error, msg) { - this.logger.error(msg); - } - - /** - * Builds default localSearchAssertionsCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'localSearchAssertionsCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default LocalSearchAssertionsCommand; diff --git a/src/commands/protocols/search/sender/search-assertions-command.js b/src/commands/protocols/search/sender/search-assertions-command.js deleted file mode 100644 index 36932d792b..0000000000 --- a/src/commands/protocols/search/sender/search-assertions-command.js +++ /dev/null @@ -1,53 +0,0 @@ -import Command from '../../../command.js'; - -class SearchAssertionsCommand extends Command { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.commandExecutor = ctx.commandExecutor; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute(command) { - const { nodes, operationId, query, options } = command.data; - - const commandSequence = ['searchAssertionsInitCommand', 'searchAssertionsRequestCommand']; - const addCommandPromise = []; - nodes.forEach((node) => { - addCommandPromise.push( - this.commandExecutor.add({ - name: commandSequence[0], - sequence: commandSequence.slice(1), - delay: 0, - data: { operationId, node, query, options }, - transactional: false, - }), - ); - }); - - await Promise.any(addCommandPromise); - - // todo schedule timeout command - return Command.empty(); - } - - /** - * Builds default searchAssertionsCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'searchAssertionsCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default SearchAssertionsCommand; diff --git a/src/commands/protocols/search/sender/search-assertions-init-command.js b/src/commands/protocols/search/sender/search-assertions-init-command.js deleted file mode 100644 index ac0ec6e3b7..0000000000 --- a/src/commands/protocols/search/sender/search-assertions-init-command.js +++ /dev/null @@ -1,36 +0,0 @@ -import ProtocolInitCommand from '../../common/protocol-init-command.js'; -import { ERROR_TYPE, NETWORK_PROTOCOLS } from '../../../../constants/constants.js'; - -class SearchAssertionsInitCommand extends ProtocolInitCommand { - constructor(ctx) { - super(ctx); - - this.errorType = ERROR_TYPE.SEARCH_ASSERTIONS_INIT_ERROR; - this.networkProtocol = NETWORK_PROTOCOLS.SEARCH_ASSERTIONS; - } - - async prepareMessage(command) { - const { query, options } = command.data; - - return { query, options }; - } - - /** - * Builds default searchAssertionsInitCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'searchAssertionsInitCommand', - delay: 0, - period: 5000, - retries: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default SearchAssertionsInitCommand; diff --git a/src/commands/protocols/search/sender/search-assertions-request-command.js b/src/commands/protocols/search/sender/search-assertions-request-command.js deleted file mode 100644 index 7b6f470daf..0000000000 --- a/src/commands/protocols/search/sender/search-assertions-request-command.js +++ /dev/null @@ -1,70 +0,0 @@ -import ProtocolRequestCommand from '../../common/protocol-request-command.js'; -import { - ERROR_TYPE, - NETWORK_PROTOCOLS, - OPERATION_ID_STATUS, -} from '../../../../constants/constants.js'; -import Command from '../../../command.js'; - -class searchAssertionsRequestCommand extends ProtocolRequestCommand { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.config = ctx.config; - this.operationIdService = ctx.operationIdService; - this.searchService = ctx.searchService; - - this.commandName = 'searchAssertionsRequestCommand'; - this.errorType = ERROR_TYPE.SEARCH_ASSERTIONS_REQUEST_ERROR; - this.networkProtocol = NETWORK_PROTOCOLS.SEARCH_ASSERTIONS; - } - - async prepareMessage(command) { - const { operationId, query, options } = command.data; - - return { - options, - operationId, - query, - }; - } - - async handleAck(command, responseData) { - await this.searchService.processSearchResponse( - command, - responseData, - OPERATION_ID_STATUS.SEARCH_ASSERTIONS.COMPLETED, - ); - return Command.empty(); - } - - async markResponseAsFailed(command, errorMessage) { - await this.searchService.processSearchResponse( - command, - OPERATION_ID_STATUS.SEARCH_ASSERTIONS.FAILED, - errorMessage, - ); - } - - handleError(operationId, error, msg) { - this.logger.error(msg); - } - - /** - * Builds default searchAssertionsRequest - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'searchAssertionsRequestCommand', - delay: 0, - transactional: false, - errorType: ERROR_TYPE.SEARCH_ASSERTIONS_REQUEST_ERROR, - }; - Object.assign(command, map); - return command; - } -} - -export default searchAssertionsRequestCommand; diff --git a/src/commands/query/query-command.js b/src/commands/query/query-command.js index 8ea9f3c41d..68860027ec 100644 --- a/src/commands/query/query-command.js +++ b/src/commands/query/query-command.js @@ -1,16 +1,27 @@ import Command from '../command.js'; -import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../constants/constants.js'; +import { + TRIPLE_STORE_REPOSITORIES, + QUERY_TYPES, + OPERATION_ID_STATUS, + ERROR_TYPE, +} from '../../constants/constants.js'; class QueryCommand extends Command { constructor(ctx) { super(ctx); - this.queryService = ctx.queryService; + this.dataService = ctx.dataService; + this.tripleStoreService = ctx.tripleStoreService; this.errorType = ERROR_TYPE.QUERY.LOCAL_QUERY_ERROR; } async execute(command) { - const { query, queryType, operationId } = command.data; + const { + query, + queryType, + operationId, + repository = TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT, + } = command.data; let data; @@ -19,7 +30,20 @@ class QueryCommand extends Command { OPERATION_ID_STATUS.QUERY.QUERY_START, ); try { - data = await this.queryService.query(query, queryType); + switch (queryType) { + case QUERY_TYPES.CONSTRUCT: { + data = await this.tripleStoreService.construct(repository, query); + break; + } + case QUERY_TYPES.SELECT: { + data = await this.dataService.parseBindings( + await this.tripleStoreService.select(repository, query), + ); + break; + } + default: + throw new Error(`Unknown query type ${queryType}`); + } await this.operationIdService.updateOperationIdStatus( operationId, diff --git a/src/constants/constants.js b/src/constants/constants.js index ec0075f78c..759a50bffe 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -34,6 +34,22 @@ export const DHT_TYPES = { DUAL: 'dual', WAN: 'wan', LAN: 'lan' }; export const PEER_OFFLINE_LIMIT = 24 * 60 * 60 * 1000; +export const TRIPLE_STORE_REPOSITORIES = { + PUBLIC_CURRENT: 'publicCurrent', + PUBLIC_HISTORY: 'publicHistory', + PRIVATE_CURRENT: 'privateCurrent', + PRIVATE_HISTORY: 'privateHistory', +}; + +export const REQUIRED_MODULES = [ + 'repository', + 'httpClient', + 'network', + 'validation', + 'blockchain', + 'tripleStore', +]; + /** * Triple store media types * @type {{APPLICATION_JSON: string, N_QUADS: string, SPARQL_RESULTS_JSON: string, LD_JSON: string}} @@ -174,6 +190,9 @@ export const ERROR_TYPE = { GET_REQUEST_REMOTE_ERROR: 'GetRequestRemoteError', GET_ERROR: 'GetError', }, + LOCAL_STORE: { + LOCAL_STORE_ERROR: 'LocalStoreError', + }, QUERY: { LOCAL_QUERY_ERROR: 'LocalQueryError', }, @@ -252,19 +271,23 @@ export const OPERATION_ID_STATUS = { VALIDATING_QUERY: 'VALIDATING_QUERY', SEARCHING_ENTITIES: 'SEARCHING_ENTITIES', }, - QUERY: { QUERY_INIT_START: 'QUERY_INIT_START', QUERY_INIT_END: 'QUERY_INIT_END', QUERY_START: 'QUERY_START', QUERY_END: 'QUERY_END', }, + LOCAL_STORE: { + LOCAL_STORE_INIT_START: 'LOCAL_STORE_INIT_START', + LOCAL_STORE_INIT_END: 'LOCAL_STORE_INIT_END', + LOCAL_STORE_START: 'LOCAL_STORE_START', + LOCAL_STORE_END: 'LOCAL_STORE_END', + }, }; export const OPERATIONS = { PUBLISH: 'publish', GET: 'get', - SEARCH: 'search', }; /** diff --git a/src/controllers/http-api/http-api-router.js b/src/controllers/http-api/http-api-router.js index d82b0468fa..767c002711 100644 --- a/src/controllers/http-api/http-api-router.js +++ b/src/controllers/http-api/http-api-router.js @@ -5,7 +5,8 @@ class HttpApiRouter { this.getHttpApiController = ctx.getHttpApiController; this.publishHttpApiController = ctx.publishHttpApiController; - this.searchHttpApiController = ctx.searchHttpApiController; + this.localStoreHttpApiController = ctx.localStoreHttpApiController; + this.queryHttpApiController = ctx.queryHttpApiController; this.resultHttpApiController = ctx.resultHttpApiController; this.infoHttpApiController = ctx.infoHttpApiController; this.bidSuggestionHttpApiController = ctx.bidSuggestionHttpApiController; @@ -32,11 +33,19 @@ class HttpApiRouter { this.httpClientModuleManager.post( '/query', (req, res) => { - this.searchHttpApiController.handleQueryRequest(req, res); + this.queryHttpApiController.handleQueryRequest(req, res); }, { requestSchema: this.jsonSchemaService.querySchema() }, ); + this.httpClientModuleManager.post( + '/local-store', + (req, res) => { + this.localStoreHttpApiController.handleLocalStoreRequest(req, res); + }, + { requestSchema: this.jsonSchemaService.localStoreSchema() }, + ); + this.httpClientModuleManager.post( '/get', (req, res) => { diff --git a/src/controllers/http-api/local-store-http-api-controller.js b/src/controllers/http-api/local-store-http-api-controller.js new file mode 100644 index 0000000000..565391fff5 --- /dev/null +++ b/src/controllers/http-api/local-store-http-api-controller.js @@ -0,0 +1,46 @@ +import BaseController from './base-http-api-controller.js'; + +import { OPERATION_ID_STATUS } from '../../constants/constants.js'; + +class LocalStoreController extends BaseController { + constructor(ctx) { + super(ctx); + this.commandExecutor = ctx.commandExecutor; + this.operationIdService = ctx.operationIdService; + } + + async handleLocalStoreRequest(req, res) { + const operationId = await this.operationIdService.generateOperationId( + OPERATION_ID_STATUS.LOCAL_STORE.LOCAL_STORE_INIT_START, + ); + + this.returnResponse(res, 202, { + operationId, + }); + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.LOCAL_STORE.LOCAL_STORE_INIT_END, + ); + + this.logger.info( + `Received assertion with assertion ids: ${req.body.map( + (reqObject) => reqObject.assertionId, + )}. Operation id: ${operationId}`, + ); + + await this.operationIdService.cacheOperationIdData(operationId, req.body); + + await this.commandExecutor.add({ + name: 'localStoreCommand', + sequence: [], + delay: 0, + data: { + operationId, + }, + transactional: false, + }); + } +} + +export default LocalStoreController; diff --git a/src/controllers/http-api/publish-http-api-controller.js b/src/controllers/http-api/publish-http-api-controller.js index 51d871c84b..c6f399efb2 100644 --- a/src/controllers/http-api/publish-http-api-controller.js +++ b/src/controllers/http-api/publish-http-api-controller.js @@ -44,12 +44,9 @@ class PublishController extends BaseController { `Received asset with assertion id: ${assertionId}, blockchain: ${blockchain}, hub contract: ${contract}, token id: ${tokenId}`, ); - const commandSequence = req.body.localStore ? ['localStoreCommand'] : []; - commandSequence.push('networkPublishCommand'); - await this.commandExecutor.add({ name: 'validateAssertionCommand', - sequence: commandSequence, + sequence: ['networkPublishCommand'], delay: 0, period: 5000, retries: 3, diff --git a/src/controllers/http-api/query-http-api-controller.js b/src/controllers/http-api/query-http-api-controller.js new file mode 100644 index 0000000000..02265ecb57 --- /dev/null +++ b/src/controllers/http-api/query-http-api-controller.js @@ -0,0 +1,38 @@ +import BaseController from './base-http-api-controller.js'; + +import { OPERATION_ID_STATUS } from '../../constants/constants.js'; + +class QueryController extends BaseController { + constructor(ctx) { + super(ctx); + this.commandExecutor = ctx.commandExecutor; + this.operationIdService = ctx.operationIdService; + } + + async handleQueryRequest(req, res) { + const { query, type: queryType } = req.body; + + const operationId = await this.operationIdService.generateOperationId( + OPERATION_ID_STATUS.QUERY.QUERY_INIT_START, + ); + + this.returnResponse(res, 202, { + operationId, + }); + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.QUERY.QUERY_INIT_END, + ); + + await this.commandExecutor.add({ + name: 'queryCommand', + sequence: [], + delay: 0, + data: { query, queryType, operationId }, + transactional: false, + }); + } +} + +export default QueryController; diff --git a/src/controllers/http-api/request-schema/local-store-schema.js b/src/controllers/http-api/request-schema/local-store-schema.js new file mode 100644 index 0000000000..704c72f966 --- /dev/null +++ b/src/controllers/http-api/request-schema/local-store-schema.js @@ -0,0 +1,22 @@ +export default () => ({ + type: 'array', + items: { + type: 'object', + required: ['assertionId', 'assertion'], + properties: { + assertionId: { + type: 'string', + minLength: 66, + maxLength: 66, + }, + assertion: { + type: 'array', + items: { + type: 'string', + }, + minItems: 1, + }, + }, + }, + minItems: 1, +}); diff --git a/src/controllers/http-api/request-schema/publish-schema.js b/src/controllers/http-api/request-schema/publish-schema.js index f1ef4f378e..7f756cbabd 100644 --- a/src/controllers/http-api/request-schema/publish-schema.js +++ b/src/controllers/http-api/request-schema/publish-schema.js @@ -1,98 +1,30 @@ -import { PUBLISH_TYPES } from '../../../constants/constants.js'; - -const assertionSchemaProperties = (blockchainImplementationNames) => ({ - assertionId: { - type: 'string', - minLength: '1', - }, - assertion: { - type: 'array', - items: { - type: 'string', - }, - minItems: 1, - }, - blockchain: { - enum: blockchainImplementationNames, - }, - contract: { - type: 'string', - minLength: 1, - }, - localStore: { - type: 'boolean', - }, -}); - -const assertionSchemaRequired = ['assertionId', 'assertion', 'blockchain', 'contract']; - -const assetSchemaProperties = (blockchainImplementationNames) => ({ - ...assertionSchemaProperties(blockchainImplementationNames), - tokenId: { - type: 'number', - minimum: 0, - }, -}); - -const assetSchemaRequired = [...assertionSchemaRequired, 'tokenId']; - -const indexSchemaProperties = (blockchainImplementationNames) => ({ - ...assetSchemaProperties(blockchainImplementationNames), - assertion: { - type: 'array', - items: { - type: 'string', - }, - minLength: 1, - maxLength: 5, - }, - keywords: { - type: 'array', - minLength: 1, - maxLength: 3, - uniqueItems: true, - }, -}); - -const indexSchemaRequired = [...assetSchemaRequired, 'keywords']; - -const publishSchema = (blockchainImplementationNames) => ({ +export default (blockchainImplementationNames) => ({ type: 'object', - required: ['publishType'], + required: ['assertionId', 'assertion', 'blockchain', 'contract', 'tokenId'], properties: { - publishType: { - enum: Object.values(PUBLISH_TYPES), + assertionId: { + type: 'string', + minLength: 66, + maxLength: 66, }, - }, - allOf: [ - { - if: { - properties: { publishType: { const: PUBLISH_TYPES.ASSERTION } }, - }, - then: { - properties: assertionSchemaProperties(blockchainImplementationNames), - required: assertionSchemaRequired, + assertion: { + type: 'array', + items: { + type: 'string', }, + minItems: 1, }, - { - if: { - properties: { publishType: { const: PUBLISH_TYPES.ASSET } }, - }, - then: { - properties: assetSchemaProperties(blockchainImplementationNames), - required: assetSchemaRequired, - }, + blockchain: { + enum: blockchainImplementationNames, }, - { - if: { - properties: { publishType: { const: PUBLISH_TYPES.INDEX } }, - }, - then: { - properties: indexSchemaProperties(blockchainImplementationNames), - required: indexSchemaRequired, - }, + contract: { + type: 'string', + minLength: 42, + maxLength: 42, + }, + tokenId: { + type: 'number', + minimum: 0, }, - ], + }, }); - -export default publishSchema; diff --git a/src/controllers/http-api/request-schema/search-schema.js b/src/controllers/http-api/request-schema/search-schema.js deleted file mode 100644 index 366800b018..0000000000 --- a/src/controllers/http-api/request-schema/search-schema.js +++ /dev/null @@ -1,20 +0,0 @@ -export default () => ({ - type: 'object', - required: ['keywords', 'limit', 'offset'], - properties: { - keywords: { - type: 'array', - minItems: 1, - maxItems: 3, - uniqueItems: true, - }, - limit: { - type: 'number', - minimum: 1, - }, - offset: { - type: 'number', - minimum: 0, - }, - }, -}); diff --git a/src/controllers/http-api/result-http-api-controller.js b/src/controllers/http-api/result-http-api-controller.js index aa88b1e804..017f0c8687 100644 --- a/src/controllers/http-api/result-http-api-controller.js +++ b/src/controllers/http-api/result-http-api-controller.js @@ -1,12 +1,11 @@ import { OPERATION_ID_STATUS } from '../../constants/constants.js'; import BaseController from './base-http-api-controller.js'; -const availableOperations = ['publish', 'get', 'assertions:search', 'entities:search', 'query']; +const availableOperations = ['publish', 'get', 'query', 'local-store']; class ResultController extends BaseController { constructor(ctx) { super(ctx); - this.logger = ctx.logger; this.operationIdService = ctx.operationIdService; } @@ -38,11 +37,10 @@ class ResultController extends BaseController { } switch (operation) { - case 'assertions:search': - case 'entities:search': case 'get': case 'publish': case 'query': + case 'local-store': if (handlerRecord.status === OPERATION_ID_STATUS.COMPLETED) { response.data = await this.operationIdService.getCachedOperationIdData( operationId, diff --git a/src/controllers/http-api/search-http-api-controller.js b/src/controllers/http-api/search-http-api-controller.js deleted file mode 100644 index a145fbc6a2..0000000000 --- a/src/controllers/http-api/search-http-api-controller.js +++ /dev/null @@ -1,156 +0,0 @@ -import BaseController from './base-http-api-controller.js'; - -import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS } from '../../constants/constants.js'; - -class SearchController extends BaseController { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.fileService = ctx.fileService; - this.commandExecutor = ctx.commandExecutor; - this.operationIdService = ctx.operationIdService; - this.queryService = ctx.queryService; - } - - async handleSearchAssertionsRequest(req, res) { - const { query } = req.query; - - const operationId = await this.operationIdService.generateOperationId( - OPERATION_ID_STATUS.SEARCH_ASSERTIONS.SEARCH_START, - ); - - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.SEARCH_ASSERTIONS.VALIDATING_QUERY, - ); - - this.returnResponse(res, 202, { - operationId, - }); - - this.logger.info( - `Search assertions for ${query} with operation id ${operationId} initiated.`, - ); - - try { - // TODO: updated with query params from get req - const options = { - prefix: true, - limit: 40, - }; - - const commandData = { - operationId, - query, - options, - networkProtocol: NETWORK_PROTOCOLS.SEARCH_ASSERTIONS, - }; - - const commandSequence = [ - 'localSearchAssertionsCommand', - 'findNodesCommand', - 'searchAssertionsCommand', - ]; - - await this.commandExecutor.add({ - name: commandSequence[0], - sequence: commandSequence.slice(1), - delay: 0, - data: commandData, - transactional: false, - }); - } catch (error) { - this.logger.error( - `Error while initializing search for assertions: ${error.message}. ${error.stack}`, - ); - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.FAILED, - 'Unable to search for assertions, Failed to process input data!', - ); - } - } - - async handleSearchEntitiesRequest(req, res) { - const { query } = req.query; - - const operationId = await this.operationIdService.generateOperationId( - OPERATION_ID_STATUS.SEARCH_ENTITIES.VALIDATING_QUERY, - ); - - this.returnResponse(res, 202, { - operationId, - }); - - this.logger.info( - `Search entities for ${query} with operation id ${operationId} initiated.`, - ); - - try { - // TODO: updated with query params - const options = { - prefix: true, - limit: 40, - }; - - const commandData = { - operationId, - query, - options, - networkProtocol: NETWORK_PROTOCOLS.SEARCH, - }; - - const commandSequence = [ - 'localSearchEntitiesCommand', - 'findNodesCommand', - 'searchEntitiesCommand', - ]; - - await this.commandExecutor.add({ - name: commandSequence[0], - sequence: commandSequence.slice(1), - delay: 0, - data: commandData, - transactional: false, - }); - } catch (error) { - this.logger.error( - `Error while initializing search for entities: ${error.message}. ${error.stack}`, - ); - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.FAILED, - 'Unable to search for entities, Failed to process input data!', - ); - } - } - - async handleQueryRequest(req, res) { - const { query, type: queryType } = req.body; - - const operationId = await this.operationIdService.generateOperationId( - OPERATION_ID_STATUS.QUERY.QUERY_INIT_START, - ); - - this.returnResponse(res, 202, { - operationId, - }); - - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.QUERY.QUERY_INIT_END, - ); - - const commandSequence = ['queryCommand']; - - await this.commandExecutor.add({ - name: commandSequence[0], - sequence: commandSequence.slice(1), - delay: 0, - data: { query, queryType, operationId }, - transactional: false, - }); - } -} - -export default SearchController; diff --git a/src/controllers/rpc/search-rpc-controller.js b/src/controllers/rpc/search-rpc-controller.js deleted file mode 100644 index cbda9619e3..0000000000 --- a/src/controllers/rpc/search-rpc-controller.js +++ /dev/null @@ -1,36 +0,0 @@ -import BaseController from './base-rpc-controller.js'; - -import { NETWORK_MESSAGE_TYPES } from '../../constants/constants.js'; - -class SearchController extends BaseController { - constructor(ctx) { - super(ctx); - this.commandExecutor = ctx.commandExecutor; - } - - async v1_0_0HandleRequest(message, remotePeerId) { - let commandName; - const { operationId } = message.header; - const commandData = { message, remotePeerId, operationId }; - switch (message.header.messageType) { - case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_INIT: - commandName = 'handleSearchEntitiesInitCommand'; - break; - case NETWORK_MESSAGE_TYPES.REQUESTS.PROTOCOL_REQUEST: - commandName = 'handleSearchEntitiesRequestCommand'; - break; - default: - throw Error('unknown messageType'); - } - - await this.commandExecutor.add({ - name: commandName, - sequence: [], - delay: 0, - data: commandData, - transactional: false, - }); - } -} - -export default SearchController; diff --git a/src/migration/triple-store-user-configuration-migration.js b/src/migration/triple-store-user-configuration-migration.js new file mode 100644 index 0000000000..ce9656f93a --- /dev/null +++ b/src/migration/triple-store-user-configuration-migration.js @@ -0,0 +1,134 @@ +import path from 'path'; +import appRootPath from 'app-root-path'; +import BaseMigration from './base-migration.js'; + +class TripleStoreUserConfigurationMigration extends BaseMigration { + async executeMigration() { + if (process.env.NODE_ENV !== 'development' && process.env.NODE_ENV !== 'test') { + const configurationFolderPath = path.join(appRootPath.path, '..'); + const configurationFilePath = path.join( + configurationFolderPath, + this.config.configFilename, + ); + + const userConfiguration = await this.fileService.loadJsonFromFile( + configurationFilePath, + ); + if (userConfiguration.modules.tripleStore.implementation) { + for (const implementationName in userConfiguration.modules.tripleStore + .implementation) { + const oldImplementationConfig = + userConfiguration.modules.tripleStore.implementation[implementationName] + .config; + if (oldImplementationConfig && !oldImplementationConfig.repositories) { + let { url, username, password, repository } = oldImplementationConfig; + + if (!url) { + url = + implementationName === 'ot-blazegraph' + ? 'http://localhost:9999' + : 'http://localhost:3030'; + } + + if (!username) { + username = 'admin'; + } + + if (!password) { + password = ''; + } + + if (!repository) { + if (implementationName === 'ot-blazegraph') { + repository = 'kb'; + } + if (implementationName === 'ot-fuseki') { + repository = 'node0'; + } + } + const newImplementationConfig = { + repositories: { + publicCurrent: { + url, + name: repository, + username, + password, + }, + publicHistory: { + url, + name: 'public-history', + username, + password, + }, + privateCurrent: { + url, + name: 'private-current', + username, + password, + }, + privateHistory: { + url, + name: 'private-history', + username, + password, + }, + }, + }; + + userConfiguration.modules.tripleStore.implementation[ + implementationName + ].config = newImplementationConfig; + } + } + } else { + const configurationTemplatePath = path.join( + appRootPath.path, + 'tools', + 'local-network-setup', + '.origintrail_noderc_template.json', + ); + const configurationTemplate = await this.fileService.loadJsonFromFile( + configurationTemplatePath, + ); + + if ( + userConfiguration.modules.tripleStore.defaultImplementation === 'ot-blazegraph' + ) { + userConfiguration.modules.tripleStore.implementation = { + 'ot-blazegraph': + configurationTemplate.modules.tripleStore.implementation[ + 'ot-blazegraph' + ], + }; + configurationTemplate.modules.tripleStore.implementation[ + 'ot-blazegraph' + ].enabled = true; + configurationTemplate.modules.tripleStore.implementation[ + 'ot-blazegraph' + ].config.repositories.publicCurrent.name = 'kb'; + } else if ( + userConfiguration.modules.tripleStore.defaultImplementation === 'ot-fuseki' + ) { + userConfiguration.modules.tripleStore.implementation = { + 'ot-fuseki': + configurationTemplate.modules.tripleStore.implementation['ot-fuseki'], + }; + configurationTemplate.modules.tripleStore.implementation[ + 'ot-fuseki' + ].enabled = true; + configurationTemplate.modules.tripleStore.implementation[ + 'ot-fuseki' + ].config.repositories.publicCurrent.name = 'node0'; + } + } + delete userConfiguration.modules.tripleStore.defaultImplementation; + await this.fileService.writeContentsToFile( + configurationFolderPath, + this.config.configFilename, + JSON.stringify(userConfiguration, null, 4), + ); + } + } +} + +export default TripleStoreUserConfigurationMigration; diff --git a/src/modules/base-module-manager.js b/src/modules/base-module-manager.js index 1650974390..e96161999c 100644 --- a/src/modules/base-module-manager.js +++ b/src/modules/base-module-manager.js @@ -1,36 +1,22 @@ -const requiredModules = [ - 'repository', - 'httpClient', - 'network', - 'validation', - 'blockchain', - 'tripleStore', -]; +import ModuleConfigValidation from './module-config-validation.js'; +import { REQUIRED_MODULES } from '../constants/constants.js'; class BaseModuleManager { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; + + this.moduleConfigValidation = new ModuleConfigValidation(ctx); } async initialize() { try { const moduleConfig = this.config.modules[this.getName()]; - if (!moduleConfig || !moduleConfig.enabled) { - const message = `${this.getName()} module not defined or enabled in configuration`; - if (requiredModules.includes(this.getName())) { - throw new Error(`${message} but it's required!`); - } - this.logger.warn(message); - return false; - } + this.moduleConfigValidation.validateModule(this.getName(), moduleConfig); this.handlers = {}; for (const implementationName in moduleConfig.implementation) { - if ( - moduleConfig.defaultImplementation && - implementationName !== moduleConfig.defaultImplementation - ) { + if (!moduleConfig.implementation[implementationName].enabled) { // eslint-disable-next-line no-continue continue; } @@ -54,7 +40,6 @@ class BaseModuleManager { const module = new ModuleClass(); // eslint-disable-next-line no-await-in-loop await module.initialize(implementationConfig.config, this.logger); - module.getImplementationName = () => implementationName; this.logger.info( @@ -65,13 +50,11 @@ class BaseModuleManager { config: implementationConfig.config, }; } - if (Object.keys(this.handlers).length === 0) { - throw new Error(`No implementation initialized for module: ${this.getName()}.`); - } this.initialized = true; + return true; } catch (error) { - if (requiredModules.includes(this.getName())) { + if (REQUIRED_MODULES.includes(this.getName())) { throw new Error( `${this.getName()} module is required but got error during initialization - ${ error.message @@ -79,6 +62,7 @@ class BaseModuleManager { ); } this.logger.error(error.message); + return false; } } diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index f207b253e6..f451c10ead 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -544,19 +544,24 @@ class Web3Service { } async getAssertionIdByIndex(assetContractAddress, tokenId, index) { - return this.callContractFunction( - this.assetStorageContracts[assetContractAddress.toLowerCase()], - 'getAssertionIdByIndex', - [tokenId, index], - ); + const assetStorageContractInstance = + this.assetStorageContracts[assetContractAddress.toLowerCase()]; + if (!assetStorageContractInstance) throw Error('Unknown asset storage contract address'); + + return this.callContractFunction(assetStorageContractInstance, 'getAssertionIdByIndex', [ + tokenId, + index, + ]); } async getLatestAssertionId(assetContractAddress, tokenId) { - return this.callContractFunction( - this.assetStorageContracts[assetContractAddress.toLowerCase()], - 'getLatestAssertionId', - [tokenId], - ); + const assetStorageContractInstance = + this.assetStorageContracts[assetContractAddress.toLowerCase()]; + if (!assetStorageContractInstance) throw Error('Unknown asset storage contract address'); + + return this.callContractFunction(assetStorageContractInstance, 'getLatestAssertionId', [ + tokenId, + ]); } async getAssertionIssuer(assertionId) { diff --git a/src/modules/module-config-validation.js b/src/modules/module-config-validation.js new file mode 100644 index 0000000000..4eaaf02b42 --- /dev/null +++ b/src/modules/module-config-validation.js @@ -0,0 +1,73 @@ +import { REQUIRED_MODULES, TRIPLE_STORE_REPOSITORIES } from '../constants/constants.js'; + +class ModuleConfigValidation { + constructor(ctx) { + this.config = ctx.config; + this.logger = ctx.logger; + } + + validateModule(name, config) { + this.validateRequiredModule(name, config); + + const capitalizedName = name.charAt(0).toUpperCase() + name.slice(1); + this[`validate${capitalizedName}`](config); + } + + validateAutoUpdater() { + return true; + } + + validateBlockchain() { + return true; + } + + validateHttpClient() { + return true; + } + + validateNetwork() { + return true; + } + + validateRepository() { + return true; + } + + validateTripleStore(config) { + const occurences = {}; + for (const implementation of Object.values(config.implementation)) { + // eslint-disable-next-line no-continue + if (!implementation.enabled) continue; + + for (const repository in implementation.config.repositories) { + if (!occurences[repository]) occurences[repository] = 0; + occurences[repository] += 1; + } + } + for (const repository of Object.values(TRIPLE_STORE_REPOSITORIES)) { + if (occurences[repository] !== 1) + throw Error(`Exactly one config for repository ${repository} needs to be defined.`); + } + } + + validateValidation() { + return true; + } + + validateRequiredModule(moduleName, moduleConfig) { + if ( + !moduleConfig?.enabled || + !Object.values(moduleConfig.implementation).filter( + (implementationConfig) => implementationConfig.enabled, + ).length + ) { + const message = `${moduleName} module not defined or enabled in configuration`; + if (REQUIRED_MODULES.includes(moduleName)) { + throw new Error(`${message} but it's required!`); + } + this.logger.warn(message); + } + } +} + +export default ModuleConfigValidation; diff --git a/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js b/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js index 5b95528b8b..47a3291c6d 100644 --- a/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js +++ b/src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js @@ -1,15 +1,59 @@ import axios from 'axios'; +import jsonld from 'jsonld'; import OtTripleStore from '../ot-triple-store.js'; class OtBlazegraph extends OtTripleStore { - initializeSparqlEndpoints(url) { - this.sparqlEndpoint = `${url}/sparql`; - this.sparqlEndpointUpdate = `${url}/sparql`; + async initialize(config, logger) { + await super.initialize(config, logger); + + await Promise.all( + Object.keys(this.repositories).map(async (repository) => { + const { url, name } = this.repositories[repository]; + + if (!(await this.repositoryExists(repository))) { + await axios.post( + `${url}/blazegraph/namespace`, + `com.bigdata.rdf.sail.truthMaintenance=false\ncom.bigdata.namespace.${name}.lex.com.bigdata.btree.BTree.branchingFactor=400\ncom.bigdata.rdf.store.AbstractTripleStore.textIndex=false\ncom.bigdata.rdf.store.AbstractTripleStore.justify=false\ncom.bigdata.namespace.${name}.spo.com.bigdata.btree.BTree.branchingFactor=1024\ncom.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=false\ncom.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms\ncom.bigdata.rdf.sail.namespace=${name}\ncom.bigdata.rdf.store.AbstractTripleStore.quads=true\ncom.bigdata.rdf.store.AbstractTripleStore.geoSpatial=false\ncom.bigdata.journal.Journal.groupCommit=false\ncom.bigdata.rdf.sail.isolatableIndices=false\n`, + { + headers: { + 'Content-Type': 'text/plain', + }, + }, + ); + } + }), + ); + } + + initializeSparqlEndpoints(repository) { + const { url, name } = this.repositories[repository]; + this.repositories[repository].sparqlEndpoint = `${url}/blazegraph/namespace/${name}/sparql`; + this.repositories[ + repository + ].sparqlEndpointUpdate = `${url}/blazegraph/namespace/${name}/sparql`; } - async healthCheck() { + async insertAssertion(repository, assertionId, assertionNquads) { + const exists = await this.assertionExists(repository, assertionId); + + if (!exists) { + return axios({ + method: 'post', + url: `${this.repositories[repository].sparqlEndpointUpdate}?context-uri=assertion:${assertionId}`, + headers: { + 'Content-Type': 'text/x-nquads', + }, + data: assertionNquads, + }); + } + } + + async healthCheck(repository) { try { - const response = await axios.get(`${this.config.url}/status`, {}); + const response = await axios.get( + `${this.repositories[repository].url}/blazegraph/status`, + {}, + ); if (response.data !== null) { return true; } @@ -19,6 +63,53 @@ class OtBlazegraph extends OtTripleStore { } } + async deleteRepository(repository) { + const { url, name } = this.repositories[repository]; + this.logger.info( + `Deleting ${this.getName()} triple store repository: ${repository} with name: ${name}`, + ); + + if (await this.repositoryExists(repository)) { + await axios + .delete(`${url}/blazegraph/namespace/${name}`, {}) + .catch((e) => + this.logger.error( + `Error while deleting ${this.getName()} triple store repository: ${repository} with name: ${name}. Error: ${ + e.message + }`, + ), + ); + } + } + + async repositoryExists(repository) { + const { url, name } = this.repositories[repository]; + + try { + const { data: jsonldNamespaces } = await axios.get(`${url}/blazegraph/namespace`, { + params: { + 'describe-each-named-graph': 'false', + }, + headers: { + Accept: 'application/ld+json', + }, + }); + + const compactedNamespaces = await jsonld.frame(jsonldNamespaces, {}); + + return compactedNamespaces['@graph'].filter( + (namespace) => + namespace['http://www.bigdata.com/rdf#/features/KB/Namespace'] === name, + ).length; + } catch (error) { + this.logger.error( + `Error while getting ${this.getName()} repositories. Error: ${error.message}`, + ); + + return false; + } + } + getName() { return 'OtBlazegraph'; } diff --git a/src/modules/triple-store/implementation/ot-fuseki/ot-fuseki.js b/src/modules/triple-store/implementation/ot-fuseki/ot-fuseki.js index 684c234b60..1f499d3978 100644 --- a/src/modules/triple-store/implementation/ot-fuseki/ot-fuseki.js +++ b/src/modules/triple-store/implementation/ot-fuseki/ot-fuseki.js @@ -2,14 +2,36 @@ import axios from 'axios'; import OtTripleStore from '../ot-triple-store.js'; class OtFuseki extends OtTripleStore { - initializeSparqlEndpoints(url, repository) { - this.sparqlEndpoint = `${url}/${repository}/sparql`; - this.sparqlEndpointUpdate = `${url}/${repository}/update`; + async initialize(config, logger) { + await super.initialize(config, logger); + await Promise.all( + Object.keys(this.repositories).map(async (repository) => { + const { url, name } = this.repositories[repository]; + + if (!(await this.repositoryExists(repository))) { + await axios.post( + `${url}/$/datasets?dbName=${name}&dbType=tdb`, + {}, + { + headers: { + 'Content-Type': 'text/plain', + }, + }, + ); + } + }), + ); + } + + initializeSparqlEndpoints(repository) { + const { url, name } = this.repositories[repository]; + this.repositories[repository].sparqlEndpoint = `${url}/${name}/sparql`; + this.repositories[repository].sparqlEndpointUpdate = `${url}/${name}/update`; } - async healthCheck() { + async healthCheck(repository) { try { - const response = await axios.get(`${this.config.url}/$/ping`, {}); + const response = await axios.get(`${this.repositories[repository].url}/$/ping`, {}); if (response.data !== null) { return true; } @@ -19,6 +41,41 @@ class OtFuseki extends OtTripleStore { } } + async deleteRepository(repository) { + const { url, name } = this.repositories[repository]; + this.logger.info( + `Deleting ${this.getName()} triple store repository: ${repository} with name: ${name}`, + ); + + if (await this.repositoryExists(repository)) { + await axios + .delete(`${url}/$/datasets/${name}`, {}) + .catch((e) => + this.logger.error( + `Error while deleting ${this.getName()} triple store repository: ${repository} with name: ${name}. Error: ${ + e.message + }`, + ), + ); + } + } + + async repositoryExists(repository) { + const { url, name } = this.repositories[repository]; + try { + const response = await axios.get(`${url}/$/datasets`); + + return response.data.datasets.filter((dataset) => dataset['ds.name'] === `/${name}`) + .length; + } catch (error) { + this.logger.error( + `Error while getting ${this.getName()} repositories. Error: ${error.message}`, + ); + + return false; + } + } + getName() { return 'OtFuseki'; } diff --git a/src/modules/triple-store/implementation/ot-graphdb/ot-graphdb.js b/src/modules/triple-store/implementation/ot-graphdb/ot-graphdb.js index b182801fa3..97745cb2cd 100644 --- a/src/modules/triple-store/implementation/ot-graphdb/ot-graphdb.js +++ b/src/modules/triple-store/implementation/ot-graphdb/ot-graphdb.js @@ -2,61 +2,73 @@ import graphdb from 'graphdb'; import axios from 'axios'; import OtTripleStore from '../ot-triple-store.js'; -const { server, repository, http } = graphdb; +const { server, repository: repo, http } = graphdb; class OtGraphdb extends OtTripleStore { async initialize(config, logger) { await super.initialize(config, logger); - const serverConfig = new server.ServerClientConfig(this.config.url) - .setTimeout(40000) - .setHeaders({ - Accept: http.RDFMimeType.N_QUADS, - }) - .setKeepAlive(true); - const s = new server.GraphDBServerClient(serverConfig); - const exists = await s.hasRepository(this.config.repository); - if (!exists) { - try { - await s.createRepository( - new repository.RepositoryConfig( - this.config.repository, - '', - new Map(), - '', - 'Repo title', - repository.RepositoryType.FREE, - ), - ); - } catch (e) { - await s.createRepository( - new repository.RepositoryConfig( - this.config.repository, - '', - {}, - 'graphdb:SailRepository', - 'Repo title', - 'graphdb', - ), - ); - } - } + await Promise.all( + Object.keys(this.repositories).map(async (repository) => { + const { url, name } = this.repositories[repository]; + const serverConfig = new server.ServerClientConfig(url) + .setTimeout(40000) + .setHeaders({ + Accept: http.RDFMimeType.N_QUADS, + }) + .setKeepAlive(true); + const s = new server.GraphDBServerClient(serverConfig); + // eslint-disable-next-line no-await-in-loop + const exists = await s.hasRepository(name); + if (!exists) { + try { + // eslint-disable-next-line no-await-in-loop + await s.createRepository( + new repo.RepositoryConfig( + name, + '', + new Map(), + '', + 'Repo title', + repo.RepositoryType.FREE, + ), + ); + } catch (e) { + // eslint-disable-next-line no-await-in-loop + await s.createRepository( + new repo.RepositoryConfig( + name, + '', + {}, + 'graphdb:SailRepository', + 'Repo title', + 'graphdb', + ), + ); + } + } + }), + ); } - initializeSparqlEndpoints(url, repo) { - this.sparqlEndpoint = `${url}/repositories/${repo}`; - this.sparqlEndpointUpdate = `${url}/repositories/${repo}/statements`; + initializeSparqlEndpoints(repository) { + const { url, name } = this.repositories[repository]; + this.repositories[repository].sparqlEndpoint = `${url}/repositories/${name}`; + this.repositories[ + repository + ].sparqlEndpointUpdate = `${url}/repositories/${name}/statements`; } - async healthCheck() { + async healthCheck(repository) { + const { url, username, password } = this.repositories[repository]; try { const response = await axios.get( - `${this.config.url}/repositories/${this.config.repository}/health`, + `${url}/repositories/${repository}/health`, {}, { auth: { - username: this.config.username, - password: this.config.password, + username, + password, }, }, ); @@ -74,6 +86,28 @@ class OtGraphdb extends OtTripleStore { } } + async deleteRepository(repository) { + const { url, name } = this.repositories[repository]; + this.logger.info( + `Deleting ${this.getName()} triple store repository: ${repository} with name: ${name}`, + ); + + const serverConfig = new server.ServerClientConfig(url) + .setTimeout(40000) + .setHeaders({ + Accept: http.RDFMimeType.N_QUADS, + }) + .setKeepAlive(true); + const s = new server.GraphDBServerClient(serverConfig); + s.deleteRepository(name).catch((e) => + this.logger.warn( + `Error while deleting ${this.getName()} triple store repository: ${repository} with name: ${name}. Error: ${ + e.message + }`, + ), + ); + } + getName() { return 'GraphDB'; } diff --git a/src/modules/triple-store/implementation/ot-triple-store.js b/src/modules/triple-store/implementation/ot-triple-store.js index 99b227a568..7cc495bd31 100644 --- a/src/modules/triple-store/implementation/ot-triple-store.js +++ b/src/modules/triple-store/implementation/ot-triple-store.js @@ -9,59 +9,77 @@ import { class OtTripleStore { async initialize(config, logger) { - this.config = config; this.logger = logger; - this.initializeSparqlEndpoints(this.config.url, this.config.repository); + this.repositories = config.repositories; + this.initializeRepositories(); + this.initializeContexts(); + await this.ensureConnections(); + this.queryEngine = new Engine(); + } - let ready = await this.healthCheck(); - let retries = 0; - while (!ready && retries < TRIPLE_STORE_CONNECT_MAX_RETRIES) { - retries += 1; - this.logger.warn( - `Cannot connect to Triple store (${this.getName()}), retry number: ${retries}/${TRIPLE_STORE_CONNECT_MAX_RETRIES}. Retrying in ${TRIPLE_STORE_CONNECT_RETRY_FREQUENCY} seconds.`, - ); - /* eslint-disable no-await-in-loop */ - await setTimeout(TRIPLE_STORE_CONNECT_RETRY_FREQUENCY * 1000); - ready = await this.healthCheck(); + initializeRepositories() { + for (const repository of Object.keys(this.repositories)) { + this.initializeSparqlEndpoints(repository); } - if (retries === TRIPLE_STORE_CONNECT_MAX_RETRIES) { - this.logger.error( - `Triple Store (${this.getName()}) not available, max retries reached.`, - ); - process.exit(1); - } - - this.queryEngine = new Engine(); - this.filtertype = { - KEYWORD: 'keyword', - KEYWORDPREFIX: 'keywordPrefix', - TYPES: 'types', - ISSUERS: 'issuers', - }; - const sources = [ - { - type: 'sparql', - value: `${this.sparqlEndpoint}`, - }, - ]; - - this.insertContext = { - sources, - destination: { - type: 'sparql', - value: `${this.sparqlEndpointUpdate}`, - }, - }; - this.queryContext = { - sources, - }; } initializeSparqlEndpoints() { throw Error('initializeSparqlEndpoints not implemented'); } - async assetExists(ual, blockchain, contract, tokenId) { + async deleteRepository() { + throw Error('deleteRepository not implemented'); + } + + initializeContexts() { + for (const repository in this.repositories) { + const sources = [ + { + type: 'sparql', + value: this.repositories[repository].sparqlEndpoint, + }, + ]; + + this.repositories[repository].insertContext = { + sources, + destination: { + type: 'sparql', + value: this.repositories[repository].sparqlEndpointUpdate, + }, + }; + this.repositories[repository].queryContext = { + sources, + }; + } + } + + async ensureConnections() { + const ensureConnectionPromises = Object.keys(this.repositories).map(async (repository) => { + let ready = await this.healthCheck(repository); + let retries = 0; + while (!ready && retries < TRIPLE_STORE_CONNECT_MAX_RETRIES) { + retries += 1; + this.logger.warn( + `Cannot connect to Triple store (${this.getName()}), repository: ${repository}, located at: ${ + this.repositories[repository].url + } retry number: ${retries}/${TRIPLE_STORE_CONNECT_MAX_RETRIES}. Retrying in ${TRIPLE_STORE_CONNECT_RETRY_FREQUENCY} seconds.`, + ); + /* eslint-disable no-await-in-loop */ + await setTimeout(TRIPLE_STORE_CONNECT_RETRY_FREQUENCY * 1000); + ready = await this.healthCheck(repository); + } + if (retries === TRIPLE_STORE_CONNECT_MAX_RETRIES) { + this.logger.error( + `Triple Store (${this.getName()}) not available, max retries reached.`, + ); + process.exit(1); + } + }); + + await Promise.all(ensureConnectionPromises); + } + + async assetExists(repository, ual, blockchain, contract, tokenId) { const query = `PREFIX schema: <${SCHEMA_CONTEXT}> ASK WHERE { GRAPH { @@ -71,51 +89,74 @@ class OtTripleStore { } }`; - return this.ask(query); + return this.ask(repository, query); } - async insertAsset(ual, assertionId, assetNquads) { - // const exists = await this.assetExists(ual, assertionId) + async isAssertionIdShared(repository, assertionId) { + const query = `PREFIX schema: <${SCHEMA_CONTEXT}> + SELECT (COUNT(DISTINCT ?ual) as ?count) + WHERE { + GRAPH { + ?ual schema:assertion + } + }`; + const count = await this.select(repository, query); + return count > 1; + } - // if(!exists) { - const insertion = ` - PREFIX schema: <${SCHEMA_CONTEXT}> - DELETE { - <${ual}> schema:latestAssertion ?o . + async getAssetAssertionIds(repository, ual) { + const query = `PREFIX schema: <${SCHEMA_CONTEXT}> + SELECT DISTINCT ?assertionId + WHERE { + GRAPH { + <${ual}> schema:assertion ?assertionId . + } + }`; + return this.select(repository, query); + } + + async assetAgreementExists(repository, ual, blockchain, contract, tokenId) { + const query = `PREFIX schema: <${SCHEMA_CONTEXT}> + ASK WHERE { + GRAPH { + <${ual}> schema:blockchain "${blockchain}"; + schema:contract "${contract}"; + schema:tokenId ${tokenId}; + schema:assertion ?assertion; + schema:agreementStartTime ?agreementStartTime; + schema:agreementEndTime ?agreementEndTime; + schema:keyword ?keyword; + } + }`; + + return this.ask(repository, query); + } + + async insertAsset(repository, ual, assetNquads, deleteAssetTriples = true) { + const deleteAssetTriplesQuery = `DELETE { + <${ual}> schema:assertion ?assertion . <${ual}> schema:agreementEndTime ?agreementEndTime } WHERE { GRAPH { ?s ?p ?o . <${ual}> schema:agreementEndTime ?agreementEndTime . - <${ual}> schema:latestAssertion ?latestAssertion . - } - }; - INSERT DATA { - GRAPH { - ${assetNquads} + <${ual}> schema:assertion ?assertion . } - }`; - await this.queryEngine.queryVoid(insertion, this.insertContext); - // } - } - - async insertIndex(keyword, indexNquads, assetNquads) { + };`; const insertion = ` PREFIX schema: <${SCHEMA_CONTEXT}> + ${deleteAssetTriples ? deleteAssetTriplesQuery : ''} INSERT DATA { GRAPH { ${assetNquads} } - GRAPH { - ${indexNquads} - } }`; - await this.queryEngine.queryVoid(insertion, this.insertContext); + await this.queryEngine.queryVoid(insertion, this.repositories[repository].insertContext); } - async insertAssertion(assertionId, assertionNquads) { - const exists = await this.assertionExists(assertionId); + async insertAssertion(repository, assertionId, assertionNquads) { + const exists = await this.assertionExists(repository, assertionId); if (!exists) { const insertion = ` @@ -125,37 +166,44 @@ class OtTripleStore { ${assertionNquads} } }`; - await this.queryEngine.queryVoid(insertion, this.insertContext); + await this.queryEngine.queryVoid( + insertion, + this.repositories[repository].insertContext, + ); } } - async construct(query) { - const result = await this._executeQuery(query, MEDIA_TYPES.N_QUADS); - return result; + async construct(repository, query) { + return this._executeQuery(repository, query, MEDIA_TYPES.N_QUADS); } - async select(query) { + async select(repository, query) { // todo: add media type once bug is fixed // no media type is passed because of comunica bug // https://github.com/comunica/comunica/issues/1034 - const result = await this._executeQuery(query); + const result = await this._executeQuery(repository, query); return result ? JSON.parse(result) : []; } - async ask(query) { - const result = await this.queryEngine.queryBoolean(query, this.queryContext); - return result; + async ask(repository, query) { + return this.queryEngine.queryBoolean(query, this.repositories[repository].queryContext); } - async assertionExists(graphName) { - const escapedGraphName = this.cleanEscapeCharacter(graphName); - const query = `ASK WHERE { GRAPH { ?s ?p ?o } }`; + async assertionExists(repository, assertionId) { + const escapedAssertionId = this.cleanEscapeCharacter(assertionId); + const query = `ASK WHERE { GRAPH { ?s ?p ?o } }`; - return this.ask(query); + return this.ask(repository, query); } - async get(graphName) { - const escapedGraphName = this.cleanEscapeCharacter(graphName); + async deleteAssertion(repository, assertionId) { + const query = `DROP GRAPH `; + + await this.queryEngine.queryVoid(query, this.repositories[repository].insertContext); + } + + async getAssertion(repository, assertionId) { + const escapedGraphName = this.cleanEscapeCharacter(assertionId); const query = `PREFIX schema: <${SCHEMA_CONTEXT}> CONSTRUCT { ?s ?p ?o } @@ -167,15 +215,18 @@ class OtTripleStore { } } }`; - return this.construct(query); + return this.construct(repository, query); } async healthCheck() { return true; } - async _executeQuery(query, mediaType) { - const result = await this.queryEngine.query(query, this.queryContext); + async _executeQuery(repository, query, mediaType) { + const result = await this.queryEngine.query( + query, + this.repositories[repository].queryContext, + ); const { data } = await this.queryEngine.resultToString(result, mediaType); let response = ''; @@ -191,42 +242,6 @@ class OtTripleStore { return query.replace(/['|[\]\\]/g, '\\$&'); } - createFilterParameter(queryParameter, type) { - const queryParam = this.cleanEscapeCharacter(queryParameter); - - switch (type) { - case this.filtertype.KEYWORD: - return `FILTER (lcase(?keyword) = '${queryParam}')`; - case this.filtertype.KEYWORDPREFIX: - return `FILTER contains(lcase(?keyword),'${queryParam}')`; - case this.filtertype.ISSUERS: - return `FILTER (?issuer IN (${JSON.stringify(queryParam).slice(1, -1)}))`; - case this.filtertype.TYPES: - return `FILTER (?type IN (${JSON.stringify(queryParam).slice(1, -1)}))`; - default: - return ''; - } - } - - createLimitQuery(options) { - if (!options.limit) { - return ''; - } - const queryLimit = Number(options.limit); - if (Number.isNaN(queryLimit) || !Number.isInteger(queryLimit)) { - this.logger.error(`Failed creating Limit query: ${options.limit} is not a number`); - throw new Error('Limit is not a number'); - } else if (Number.isInteger(options.limit) && options.limit < 0) { - this.logger.error(`Failed creating Limit query: ${options.limit} is negative number`); - throw new Error('Limit is not a number'); - } - return `LIMIT ${queryLimit}`; - } - - isBoolean(param) { - return typeof param === 'boolean' || ['true', 'false'].includes(param); - } - async reinitialize() { const ready = await this.healthCheck(); if (!ready) { diff --git a/src/modules/triple-store/triple-store-module-manager.js b/src/modules/triple-store/triple-store-module-manager.js index 9d576ed026..3a30318926 100644 --- a/src/modules/triple-store/triple-store-module-manager.js +++ b/src/modules/triple-store/triple-store-module-manager.js @@ -1,63 +1,117 @@ import BaseModuleManager from '../base-module-manager.js'; class TripleStoreModuleManager extends BaseModuleManager { - getName() { - return 'tripleStore'; + async insertAsset(implementationName, repository, ual, assetNquads, deleteAssetTriples) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.insertAsset( + repository, + ual, + assetNquads, + deleteAssetTriples, + ); + } } - async insertAsset(ual, assertionId, assetNquads) { - if (this.initialized) { - return this.getImplementation().module.insertAsset(ual, assertionId, assetNquads); + async assetExists(implementationName, repository, ual, blockchain, contract, tokenId) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.assetExists( + repository, + ual, + blockchain, + contract, + tokenId, + ); } } - async assetExists(ual, blockchain, contract, tokenId) { - if (this.initialized) { - return this.getImplementation().module.assetExists(ual, blockchain, contract, tokenId); + async assetAgreementExists(implementationName, repository, ual, blockchain, contract, tokenId) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.assetAgreementExists( + repository, + ual, + blockchain, + contract, + tokenId, + ); } } - async insertAssertion(assertionId, assertionNquads) { - if (this.initialized) { - return this.getImplementation().module.insertAssertion(assertionId, assertionNquads); + async insertAssertion(implementationName, repository, assertionId, assertionNquads) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.insertAssertion( + repository, + assertionId, + assertionNquads, + ); } } - async insertIndex(keyword, indexNquads, assetNquads) { - if (this.initialized) { - return this.getImplementation().module.insertIndex(keyword, indexNquads, assetNquads); + async assertionExists(implementationName, repository, uri) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.assertionExists( + repository, + uri, + ); } } - async assertionExists(uri) { - if (this.initialized) { - return this.getImplementation().module.assertionExists(uri); + async isAssertionIdShared(implementationName, repository, assertionId) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.isAssertionIdShared( + repository, + assertionId, + ); } } - async get(uri) { - if (this.initialized) { - return this.getImplementation().module.get(uri); + async getAssetAssertionIds(implementationName, repository, ual) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.getAssetAssertionIds( + repository, + ual, + ); } } - async construct(query) { - if (this.initialized) { - return this.getImplementation().module.construct(query); + async getAssertion(implementationName, repository, assertionId) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.getAssertion( + repository, + assertionId, + ); } } - async select(query) { - if (this.initialized) { - return this.getImplementation().module.select(query); + async deleteAssertion(implementationName, repository, assertionId) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.deleteAssertion( + repository, + assertionId, + ); } } - async healthCheck() { - if (this.initialized) { - return this.getImplementation().module.healthCheck(); + async construct(implementationName, repository, query) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.construct(repository, query); } } + + async select(implementationName, repository, query) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.select(repository, query); + } + } + + async deleteRepository(implementationName, repository) { + if (this.getImplementation(implementationName)) { + return this.getImplementation(implementationName).module.deleteRepository(repository); + } + } + + getName() { + return 'tripleStore'; + } } export default TripleStoreModuleManager; diff --git a/src/service/data-service.js b/src/service/data-service.js index 3c15a15dbf..1379ef64af 100644 --- a/src/service/data-service.js +++ b/src/service/data-service.js @@ -1,4 +1,3 @@ -/* eslint-disable import/extensions */ import jsonld from 'jsonld'; import { SCHEMA_CONTEXT, MEDIA_TYPES, XML_DATA_TYPES } from '../constants/constants.js'; diff --git a/src/service/get-service.js b/src/service/get-service.js index 19f0d25d37..af8ef60b8c 100644 --- a/src/service/get-service.js +++ b/src/service/get-service.js @@ -12,10 +12,6 @@ class GetService extends OperationService { constructor(ctx) { super(ctx); - this.dataService = ctx.dataService; - this.networkModuleManager = ctx.networkModuleManager; - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; - this.operationName = OPERATIONS.GET; this.networkProtocols = NETWORK_PROTOCOLS.GET; this.errorType = ERROR_TYPE.GET.GET_ERROR; @@ -95,25 +91,6 @@ class GetService extends OperationService { } } } - - async localGet(assertionId, operationId) { - this.logger.debug(`Getting assertion: ${assertionId} for operationId: ${operationId}`); - - let nquads = await this.tripleStoreModuleManager.get(assertionId); - nquads = await this.dataService.toNQuads(nquads, 'application/n-quads'); - - this.logger.debug( - `Assertion: ${assertionId} for operationId: ${operationId} ${ - nquads.length ? '' : 'not' - } found in local triple store.`, - ); - - if (nquads.length) { - this.logger.debug(`Number of n-quads retrieved from the database : ${nquads.length}`); - } - - return nquads; - } } export default GetService; diff --git a/src/service/json-schema-service.js b/src/service/json-schema-service.js index f5d6d4aa1f..d7cfa946a2 100644 --- a/src/service/json-schema-service.js +++ b/src/service/json-schema-service.js @@ -1,8 +1,8 @@ import publishSchema from '../controllers/http-api/request-schema/publish-schema.js'; import getSchema from '../controllers/http-api/request-schema/get-schema.js'; -import searchSchema from '../controllers/http-api/request-schema/search-schema.js'; import querySchema from '../controllers/http-api/request-schema/query-schema.js'; import bidSuggestionSchema from '../controllers/http-api/request-schema/bid-suggestion-schema.js'; +import localStoreSchema from '../controllers/http-api/request-schema/local-store-schema.js'; class JsonSchemaService { constructor(ctx) { @@ -21,13 +21,13 @@ class JsonSchemaService { return getSchema(); } - searchSchema() { - return searchSchema(); - } - querySchema() { return querySchema(); } + + localStoreSchema() { + return localStoreSchema(); + } } export default JsonSchemaService; diff --git a/src/service/publish-service.js b/src/service/publish-service.js index 96cc7c7fc8..c523a7a62b 100644 --- a/src/service/publish-service.js +++ b/src/service/publish-service.js @@ -1,12 +1,10 @@ import { Mutex } from 'async-mutex'; -import { formatAssertion } from 'assertion-tools'; import OperationService from './operation-service.js'; import { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, ERROR_TYPE, - SCHEMA_CONTEXT, OPERATIONS, OPERATION_REQUEST_STATUS, } from '../constants/constants.js'; @@ -14,8 +12,6 @@ import { class PublishService extends OperationService { constructor(ctx) { super(ctx); - this.ualService = ctx.ualService; - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; this.operationName = OPERATIONS.PUBLISH; this.networkProtocols = NETWORK_PROTOCOLS.STORE; @@ -95,99 +91,6 @@ class PublishService extends OperationService { } } } - - async getLatestAssertionId(blockchain, contract, tokenId) { - return this.blockchainModuleManager.getLatestAssertionId(blockchain, contract, tokenId); - } - - async localStoreIndex(assertionId, blockchain, contract, tokenId, keyword, operationId) { - const { assertion } = await this.operationIdService.getCachedOperationIdData(operationId); - const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); - - // TODO get rank from blockchain - const rank = 1; - - const indexNquads = await formatAssertion({ - '@context': SCHEMA_CONTEXT, - '@id': ual, - rank, - metadata: { '@id': `assertion:${assertionId}` }, - }); - const assetNquads = await formatAssertion({ - '@context': SCHEMA_CONTEXT, - '@id': ual, - blockchain, - contract, - tokenId, - }); - - this.logger.info( - `Inserting index for asset: ${ual}, keyword: ${keyword}, with assertion id: ${assertionId} in triple store.`, - ); - - await Promise.all( - this.tripleStoreModuleManager.insertIndex( - keyword, - indexNquads.join('\n'), - assetNquads.join('\n'), - ), - this.tripleStoreModuleManager.insertAssertion(assertionId, assertion.join('\n')), - ); - - this.logger.info( - `Index for asset: ${ual}, keyword: ${keyword}, with assertion id ${assertionId} has been successfully inserted!`, - ); - } - - async localStoreAssertion(assertionId, operationId) { - const { assertion } = await this.operationIdService.getCachedOperationIdData(operationId); - - this.logger.info(`Inserting assertion with id: ${assertionId} in triple store.`); - - await this.tripleStoreModuleManager.insertAssertion(assertionId, assertion.join('\n')); - - this.logger.info(`Assertion with id ${assertionId} has been successfully inserted!`); - } - - async localStoreAsset( - assertionId, - blockchain, - contract, - tokenId, - operationId, - agreementStartTime, - agreementEndTime, - keyword, - ) { - const { assertion } = await this.operationIdService.getCachedOperationIdData(operationId); - const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); - - const assetNquads = await formatAssertion({ - '@context': SCHEMA_CONTEXT, - '@id': ual, - blockchain, - contract, - tokenId, - assertion: { '@id': `assertion:${assertionId}` }, - latestAssertion: { '@id': `assertion:${assertionId}` }, - agreementStartTime, - agreementEndTime, - keyword, - }); - - this.logger.info( - `Inserting asset with assertion id: ${assertionId}, ual: ${ual} in triple store.`, - ); - - await Promise.all([ - this.tripleStoreModuleManager.insertAsset(ual, assertionId, assetNquads.join('\n')), - this.tripleStoreModuleManager.insertAssertion(assertionId, assertion.join('\n')), - ]); - - this.logger.info( - `Asset with assertion id: ${assertionId}, ual: ${ual} has been successfully inserted!`, - ); - } } export default PublishService; diff --git a/src/service/query-service.js b/src/service/query-service.js deleted file mode 100644 index c877ff9306..0000000000 --- a/src/service/query-service.js +++ /dev/null @@ -1,30 +0,0 @@ -import { QUERY_TYPES } from '../constants/constants.js'; - -class QueryService { - constructor(ctx) { - this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; - this.dataService = ctx.dataService; - } - - async query(query, queryType) { - switch (queryType) { - case QUERY_TYPES.CONSTRUCT: - return this.constructQuery(query); - case QUERY_TYPES.SELECT: - return this.selectQuery(query); - default: - throw new Error(`Unknown query type ${queryType}`); - } - } - - constructQuery(query) { - return this.tripleStoreModuleManager.construct(query); - } - - async selectQuery(query) { - const bindings = await this.tripleStoreModuleManager.select(query); - return this.dataService.parseBindings(bindings); - } -} - -export default QueryService; diff --git a/src/service/search-service.js b/src/service/search-service.js deleted file mode 100644 index 14af48614e..0000000000 --- a/src/service/search-service.js +++ /dev/null @@ -1,32 +0,0 @@ -import { OPERATION_ID_STATUS } from '../constants/constants.js'; - -class SearchService { - constructor(ctx) { - this.logger = ctx.logger; - - this.operationIdService = ctx.operationIdService; - this.repositoryModuleManager = ctx.repositoryModuleManager; - this.commandExecutor = ctx.commandExecutor; - } - - async processSearchResponse(command, responseData) { - const { operationId } = command.data; - - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.SEARCH_ASSERTIONS.SEARCH_END, - ); - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.SEARCH_ASSERTIONS.COMPLETED, - ); - - let data = await this.operationIdService.getCachedOperationIdData(operationId); - - data = [...new Set([...data, ...responseData])]; - - await this.operationIdService.cacheOperationIdData(operationId, data); - } -} - -export default SearchService; diff --git a/src/service/triple-store-service.js b/src/service/triple-store-service.js new file mode 100644 index 0000000000..1a35a2f856 --- /dev/null +++ b/src/service/triple-store-service.js @@ -0,0 +1,215 @@ +import { formatAssertion } from 'assertion-tools'; + +import { SCHEMA_CONTEXT, TRIPLE_STORE_REPOSITORIES } from '../constants/constants.js'; + +class TripleStoreService { + constructor(ctx) { + this.config = ctx.config; + this.logger = ctx.logger; + + this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; + this.ualService = ctx.ualService; + this.dataService = ctx.dataService; + + this.repositoryImplementations = {}; + for (const implementationName of this.tripleStoreModuleManager.getImplementationNames()) { + for (const repository in this.tripleStoreModuleManager.getImplementation( + implementationName, + ).module.repositories) { + this.repositoryImplementations[repository] = implementationName; + } + } + } + + async localStoreAssertion(assertionId, assertion, operationId) { + this.logger.info( + `Inserting assertion with id: ${assertionId} in triple store. Operation id: ${operationId}`, + ); + + await this.tripleStoreModuleManager.insertAssertion( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT], + TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT, + assertionId, + assertion.join('\n'), + ); + } + + async localStoreAsset( + assertionId, + assertion, + blockchain, + contract, + tokenId, + operationId, + agreementStartTime, + agreementEndTime, + keyword, + ) { + const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); + + this.logger.info( + `Inserting asset with assertion id: ${assertionId}, ual: ${ual} in triple store. Operation id: ${operationId}`, + ); + + /* // get current assertion, store current assertion in history repository, add triple UAL -> assertionId + const assertionIds = await this.tripleStoreModuleManager.getAssetAssertionIds( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + ual, + ); + if (assertionIds?.length) { + const currentAssertionId = assertionIds[0]; + let nquads = await this.tripleStoreModuleManager.getAssertion( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + currentAssertionId, + ); + nquads = await this.dataService.toNQuads(nquads, 'application/n-quads'); + + const historyAssetNquads = await formatAssertion({ + '@context': SCHEMA_CONTEXT, + '@id': ual, + blockchain, + contract, + tokenId, + assertion: { '@id': `assertion:${assertionId}` }, + }); + await Promise.all([ + this.tripleStoreModuleManager.insertAsset( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + ual, + historyAssetNquads.join('\n'), + false, + ), + this.tripleStoreModuleManager.insertAssertion( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + nquads, + ), + ]); + + const isAssertionIdShared = await this.tripleStoreModuleManager.isAssertionIdShared( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + currentAssertionId, + ); + if (!isAssertionIdShared) { + // delete old assertion from current repository + this.tripleStoreModuleManager.deleteAssertion( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + ); + } + } */ + + // store new assertion in current repository, update triple UAL -> assertionId + const currentAssetNquads = await formatAssertion({ + '@context': SCHEMA_CONTEXT, + '@id': ual, + blockchain, + contract, + tokenId, + assertion: { '@id': `assertion:${assertionId}` }, + agreementStartTime, + agreementEndTime, + keyword, + }); + + await Promise.all([ + this.tripleStoreModuleManager.insertAsset( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + ual, + currentAssetNquads.join('\n'), + ), + this.tripleStoreModuleManager.insertAssertion( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + assertion.join('\n'), + ), + ]); + + this.logger.info( + `Asset with assertion id: ${assertionId}, ual: ${ual} has been successfully inserted!`, + ); + } + + async localGet(assertionId, operationId, localQuery = false) { + let nquads; + if (localQuery) { + this.logger.debug( + `Getting assertion: ${assertionId} for operationId: ${operationId} from private repository`, + ); + + nquads = await this.tripleStoreModuleManager.getAssertion( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT], + TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT, + assertionId, + ); + } + if (!nquads?.length) { + this.logger.debug( + `Getting assertion: ${assertionId} for operationId: ${operationId} from public repository`, + ); + nquads = await this.tripleStoreModuleManager.getAssertion( + this.repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT], + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + ); + } + nquads = await this.dataService.toNQuads(nquads, 'application/n-quads'); + + this.logger.debug( + `Assertion: ${assertionId} for operationId: ${operationId} ${ + nquads.length ? '' : 'not' + } found in local triple store.`, + ); + + if (nquads.length) { + this.logger.debug(`Number of n-quads retrieved from the database : ${nquads.length}`); + } + + return nquads; + } + + async assetExists(repository, ual, blockchain, contract, tokenId) { + return this.tripleStoreModuleManager.assetExists( + this.repositoryImplementations[repository], + repository, + ual, + blockchain, + contract, + tokenId, + ); + } + + async assertionExists(repository, assertionId) { + return this.tripleStoreModuleManager.assertionExists( + this.repositoryImplementations[repository], + repository, + assertionId, + ); + } + + async construct(repository, query) { + return this.tripleStoreModuleManager.construct( + this.repositoryImplementations[repository], + repository, + query, + ); + } + + async select(repository, query) { + return this.tripleStoreModuleManager.select( + this.repositoryImplementations[repository], + repository, + query, + ); + } +} + +export default TripleStoreService; diff --git a/test/bdd/features/get-errors.feature b/test/bdd/features/get-errors.feature index 2712b5e74e..1789fc5a3c 100644 --- a/test/bdd/features/get-errors.feature +++ b/test/bdd/features/get-errors.feature @@ -6,19 +6,19 @@ Feature: Get errors test @get-errors Scenario: Getting non existent UAL Given I setup 4 nodes - And I wait for 10 seconds + And I wait for 2 seconds And I call get directly to ot-node 1 with nonExistentUAL And I wait for last resolve to finalize - Then Last GET operation finished with status: GetLocalError + Then Last GET operation finished with status: GetAssertionIdError #@get-errors #Scenario: GET operation result on a node with minimum replication factor greater than the number of nodes #Given I setup 4 nodes - #And I wait for 10 seconds + #And I wait for 2 seconds #And I call publish on node 1 with validAssertion #Then Last PUBLISH operation finished with status: COMPLETED #When I setup node 5 with minimumAckResponses.get set to 10 - #And I wait for 20 seconds + #And I wait for 2 seconds #And I get operation result from node 5 for last published assertion #And I wait for last resolve to finalize #Then Last GET operation finished with status: GetNetworkError diff --git a/test/bdd/features/publish-errors.feature b/test/bdd/features/publish-errors.feature index 6764127b2a..352275aca3 100644 --- a/test/bdd/features/publish-errors.feature +++ b/test/bdd/features/publish-errors.feature @@ -7,7 +7,7 @@ Feature: Publish errors test #Scenario: Publish on a node with invalid data path #Given I setup 3 nodes #And I setup node 4 with appDataPath set to \0 - #And I wait for 10 seconds + #And I wait for 2 seconds #And I call publish on node 4 with validAssertion #Then Last PUBLISH operation finished with status: PublishRouteError diff --git a/test/bdd/features/release.feature b/test/bdd/features/release.feature index 12dfa1146b..f08c18f618 100644 --- a/test/bdd/features/release.feature +++ b/test/bdd/features/release.feature @@ -7,7 +7,7 @@ Feature: Release related tests Scenario: Publishing a valid assertion Given I set R1 to be 2 Given I setup 4 nodes - And I wait for 60 seconds + And I wait for 10 seconds When I call publish on node 4 with validAssertion Then Last PUBLISH operation finished with status: COMPLETED @@ -15,12 +15,12 @@ Feature: Release related tests #@release #Scenario: Getting a result of the previously published assertion #Given I setup 4 nodes - #And I wait for 10 seconds + #And I wait for 2 seconds #When I call publish on node 4 with validAssertion #And Last PUBLISH operation finished with status: COMPLETED #And I get operation result from node 4 for last published assertion #And Last GET operation finished with status: COMPLETED #And I setup 1 additional node - #And I wait for 10 seconds + #And I wait for 2 seconds #And I get operation result from node 5 for last published assertion #Then Last GET operation finished with status: COMPLETED diff --git a/test/bdd/steps/api/datasets/requests.json b/test/bdd/steps/api/datasets/requests.json index 754bb49a19..f418a64164 100644 --- a/test/bdd/steps/api/datasets/requests.json +++ b/test/bdd/steps/api/datasets/requests.json @@ -10,7 +10,7 @@ "_:c14n0 ." ], "blockchain": "ganache", - "contract": "0x209679fA3B658Cd0fC74473aF28243bfe78a9b12", + "contract": "0x791ee543738B997B7A125bc849005B62aFD35578", "tokenId": 0 }, "blockchainNotDefinedRequestBody": { @@ -24,10 +24,10 @@ "_:c14n0 ." ], "blockchain": null, - "contract": "0x209679fA3B658Cd0fC74473aF28243bfe78a9b12", + "contract": "0x791ee543738B997B7A125bc849005B62aFD35578", "tokenId": 0 }, "nonExistentUAL": { - "id": "did:ganache:0x209679fA3B658Cd0fC74473aF28243bfe78a9b12/1" + "id": "did:ganache:0x791ee543738B997B7A125bc849005B62aFD35578/1" } } diff --git a/test/bdd/steps/api/publish.mjs b/test/bdd/steps/api/publish.mjs index a55f5208b7..98884832a8 100644 --- a/test/bdd/steps/api/publish.mjs +++ b/test/bdd/steps/api/publish.mjs @@ -1,11 +1,11 @@ import { When, Given } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { setTimeout } from 'timers/promises'; -import HttpApiHelper from "../../../utilities/http-api-helper.mjs"; -import {readFile} from "fs/promises"; +import { readFile } from 'fs/promises'; +import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; -const assertions = JSON.parse(await readFile("test/bdd/steps/api/datasets/assertions.json")); -const requests = JSON.parse(await readFile("test/bdd/steps/api/datasets/requests.json")); +const assertions = JSON.parse(await readFile('test/bdd/steps/api/datasets/assertions.json')); +const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests.json')); const httpApiHelper = new HttpApiHelper(); @@ -23,10 +23,7 @@ When( .config; const assertion = assertions[assertionName]; const result = await this.state.nodes[node - 1].client - .publish( - assertion, - { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey } - ) + .publish(assertion, { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey }) .catch((error) => { assert.fail(`Error while trying to publish assertion. ${error}`); }); diff --git a/test/bdd/steps/common.mjs b/test/bdd/steps/common.mjs index 8d0f1c20fa..7ada3563aa 100644 --- a/test/bdd/steps/common.mjs +++ b/test/bdd/steps/common.mjs @@ -2,13 +2,10 @@ import { Given } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import fs from 'fs'; import { setTimeout as sleep } from 'timers/promises'; -import { readFile } from 'fs/promises'; -import DeepExtend from 'deep-extend'; + import DkgClientHelper from '../../utilities/dkg-client-helper.mjs'; -import StepsUtils from "../../utilities/steps-utils.mjs"; +import StepsUtils from '../../utilities/steps-utils.mjs'; -const defaultConfiguration = JSON.parse(await readFile("test/bdd/steps/config/origintrail-test-node-config.json")); -const bootstrapNodeConfiguration = JSON.parse(await readFile("test/bdd/steps/config/origintrail-test-bootstrap-config.json")); const stepsUtils = new StepsUtils(); Given( @@ -24,22 +21,19 @@ Given( const wallet = wallets[nodeIndex + 1]; const managementWallet = wallets[nodeIndex + 1 + Math.floor(wallets.length / 2)]; const rpcPort = 8901 + nodeIndex; + const networkPort = 9001 + nodeIndex; const nodeName = `origintrail-test-${nodeIndex}`; const sharesTokenName = `origintrail-test-${nodeIndex}`; const sharesTokenSymbol = `OT-T-${nodeIndex}`; - const nodeConfiguration = DeepExtend( - {}, - defaultConfiguration, - stepsUtils.createNodeConfiguration( - this.state.localBlockchain, - wallet, - managementWallet, - nodeIndex, - nodeName, - rpcPort, - sharesTokenName, - sharesTokenSymbol - ), + const nodeConfiguration = stepsUtils.createNodeConfiguration( + wallet, + managementWallet, + nodeIndex, + nodeName, + rpcPort, + networkPort, + sharesTokenName, + sharesTokenSymbol, ); const forkedNode = stepsUtils.forkNode(nodeConfiguration); const logFileStream = fs.createWriteStream( @@ -53,9 +47,7 @@ Given( // eslint-disable-next-line no-loop-func forkedNode.on('message', (response) => { if (response.error) { - assert.fail( - `Error while initializing node${nodeIndex}: ${response.error}`, - ); + assert.fail(`Error while initializing node${nodeIndex}: ${response.error}`); } else { // todo if started const client = new DkgClientHelper({ @@ -88,8 +80,28 @@ Given( expect(this.state.bootstraps).to.have.length(0); expect(nodeCount).to.be.equal(1); // Currently not supported more. this.logger.log('Initializing bootstrap node'); + const nodeIndex = Object.keys(this.state.nodes).length; + const wallets = this.state.localBlockchain.getWallets(); + const wallet = wallets[nodeIndex]; + const managementWallet = + this.state.localBlockchain.getWallets()[nodeIndex + Math.floor(wallets.length / 2)]; + const rpcPort = 8900; + const networkPort = 9000; const nodeName = 'origintrail-test-bootstrap'; - const forkedNode = stepsUtils.forkNode(bootstrapNodeConfiguration); + const sharesTokenName = `${nodeName}-${nodeIndex}`; + const sharesTokenSymbol = `OT-B-${nodeIndex}`; + const nodeConfiguration = stepsUtils.createNodeConfiguration( + wallet, + managementWallet, + nodeIndex, + nodeName, + rpcPort, + networkPort, + sharesTokenName, + sharesTokenSymbol, + true, + ); + const forkedNode = stepsUtils.forkNode(nodeConfiguration); const logFileStream = fs.createWriteStream(`${this.state.scenarionLogDir}/${nodeName}.log`); forkedNode.stdout.setEncoding('utf8'); @@ -99,7 +111,7 @@ Given( }); forkedNode.on('message', async (response) => { if (response.error) { - this.logger.debug(`Error while initializing bootstrap node: ${response.error}`) + this.logger.debug(`Error while initializing bootstrap node: ${response.error}`); } else { const client = new DkgClientHelper({ endpoint: 'http://localhost', @@ -111,8 +123,8 @@ Given( this.state.bootstraps.push({ client, forkedNode, - configuration: bootstrapNodeConfiguration, - nodeRpcUrl: `http://localhost:${bootstrapNodeConfiguration.rpcPort}`, + configuration: nodeConfiguration, + nodeRpcUrl: `http://localhost:${rpcPort}`, }); } done(); @@ -124,43 +136,40 @@ Given( /^I setup node (\d+) with ([a-z][\w-]*(?:\.[\w-]+)*) set to ([^"]*)$/, { timeout: 120000 }, function setupPublishNode(nodeNum, propertyName, propertyValue, done) { - const propertyNameSplit = propertyName.split('.'); - this.logger.log(`I setup node ${nodeNum} with ${propertyName} set to ${propertyValue}`); - expect( - Object.prototype.hasOwnProperty.call(defaultConfiguration, propertyNameSplit[0]), - `Property ${propertyName} doesn't exist`, - ).to.be.equal(true); const nodeIndex = Object.keys(this.state.nodes).length; const wallets = this.state.localBlockchain.getWallets(); const wallet = wallets[nodeIndex + 1]; const managementWallet = this.state.localBlockchain.getWallets()[nodeIndex + 1 + Math.floor(wallets.length / 2)]; const rpcPort = 8901 + nodeIndex; + const networkPort = 9001 + nodeIndex; const nodeName = `origintrail-test-${nodeIndex}`; const sharesTokenName = `origintrail-test-${nodeIndex}`; const sharesTokenSymbol = `OT-T-${nodeIndex}`; - const nodeConfiguration = DeepExtend( - {}, - defaultConfiguration, - stepsUtils.createNodeConfiguration( - this.state.localBlockchain, - wallet, - managementWallet, - nodeIndex, - nodeName, - rpcPort, - sharesTokenName, - sharesTokenSymbol - ), + const nodeConfiguration = stepsUtils.createNodeConfiguration( + wallet, + managementWallet, + nodeIndex, + nodeName, + rpcPort, + networkPort, + sharesTokenName, + sharesTokenSymbol, ); + const propertyNameSplit = propertyName.split('.'); + this.logger.log(`I setup node ${nodeNum} with ${propertyName} set to ${propertyValue}`); + expect( + Object.prototype.hasOwnProperty.call(nodeConfiguration, propertyNameSplit[0]), + `Property ${propertyName} doesn't exist`, + ).to.be.equal(true); const propertyNameSplitLen = propertyNameSplit.length; let propName = nodeConfiguration; for (let i = 0; i < propertyNameSplitLen - 1; i += 1) { propName = propName[propertyNameSplit[i]]; } - if(propName[propertyNameSplit.slice(-1)] !== undefined){ + if (propName[propertyNameSplit.slice(-1)] !== undefined) { propName[propertyNameSplit.slice(-1)] = propertyValue === '\\0' ? '\0' : propertyValue; - }else{ + } else { assert.fail(`Property ${propertyName} doesn't exist`); } const forkedNode = stepsUtils.forkNode(nodeConfiguration); @@ -175,9 +184,7 @@ Given( // eslint-disable-next-line no-loop-func forkedNode.on('message', (response) => { if (response.error) { - assert.fail( - `Error while initializing node${nodeIndex} : ${response.error}`, - ); + assert.fail(`Error while initializing node${nodeIndex} : ${response.error}`); } else { const client = new DkgClientHelper({ endpoint: 'http://localhost', @@ -226,12 +233,12 @@ Given( }, ); -Given(/^I wait for (\d+) seconds$/,{ timeout: 100000}, async function waitFor(seconds) { +Given(/^I wait for (\d+) seconds$/, { timeout: 100000 }, async function waitFor(seconds) { this.logger.log(`I wait for ${seconds} seconds for nodes to connect to each other`); await sleep(seconds * 1000); -}) +}); -Given(/^I set R1 to be (\d+)$/,{ timeout: 100000}, async function waitFor(r1) { +Given(/^I set R1 to be (\d+)$/, { timeout: 100000 }, async function waitFor(r1) { this.logger.log(`I set R1 to be ${r1}`); await this.state.localBlockchain.setR1(r1); -}) +}); diff --git a/test/bdd/steps/config/origintrail-test-bootstrap-config.json b/test/bdd/steps/config/origintrail-test-bootstrap-config.json deleted file mode 100644 index 80a2ecf6ca..0000000000 --- a/test/bdd/steps/config/origintrail-test-bootstrap-config.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "modules": { - "blockchain": { - "defaultImplementation": "ganache", - "implementation": { - "ganache": { - "config": { - "blockchainTitle": "ganache", - "networkId": "ganache::testnet", - "rpcEndpoints": ["http://localhost:7545"], - "hubContractAddress": "0x209679fA3B658Cd0fC74473aF28243bfe78a9b12", - "evmOperationalWalletPublicKey": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", - "evmOperationalWalletPrivateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e", - "evmManagementWalletPublicKey": "0xBaF76aC0d0ef9a2FFF76884d54C9D3e270290a43", - "evmManagementWalletPrivateKey": "0x9b9af041edc816692276ac3c8f1d5565e3c01ddff80ec982943a29bd8d1d8863", - "sharesTokenName": "LocalNode0", - "sharesTokenSymbol": "LN0" - } - } - } - }, - "network": { - "enabled": true, - "implementation": { - "libp2p-service": { - "package": "./network/implementation/libp2p-service.js", - "config": { - "kBucketSize": 20, - "connectionManager": { - "autoDial": true, - "autoDialInterval": 10e3, - "dialTimeout": 2e3 - }, - "peerRouting": { - "refreshManager": { - "enabled": true, - "interval": 6e5, - "bootDelay": 2e3 - } - }, - "port": 9000, - "bootstrap": [], - "privateKey": "CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ=" - } - } - } - }, - "repository": { - "implementation": { - "sequelize-repository": { - "config": { - "database": "operationaldbbootstrap", - "password": "" - } - } - } - }, - "tripleStore": { - "enabled": true, - "defaultImplementation": "ot-graphdb", - "implementation": { - "ot-graphdb": { - "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", - "config": { - "url": "http://localhost:7200", - "repository": "origintrail-test-bootstrap", - "username": "admin", - "password": "" - } - } - } - }, - "validation": { - "enabled": true, - "implementation": { - "merkle-validation": { - "package": "./validation/implementation/merkle-validation.js", - "config": {} - } - } - }, - "httpClient": { - "enabled": true, - "implementation": { - "express-http-client": { - "package": "./http-client/implementation/express-http-client.js", - "config": { - "useSsl": false, - "port": 8900, - "sslKeyPath": "/root/certs/privkey.pem", - "sslCertificatePath": "/root/certs/fullchain.pem", - "rateLimiter": { - "timeWindowSeconds": 60, - "maxRequests": 10 - } - } - } - } - } - }, - "operationalDatabase": { - "databaseName": "operationaldbbootstrap" - }, - "graphDatabase": { - "name": "origintrail-test-bootstrap" - }, - "rpcPort": 8900, - "minimumAckResponses": { - "publish": 2, - "get": 1 - }, - "auth": { - "ipBasedAuthEnabled": false - } -} diff --git a/test/bdd/steps/config/origintrail-test-node-config.json b/test/bdd/steps/config/origintrail-test-node-config.json deleted file mode 100644 index 21f6863c8e..0000000000 --- a/test/bdd/steps/config/origintrail-test-node-config.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "modules": { - "blockchain": { - "defaultImplementation": "ganache", - "implementation": { - "ganache": { - "config": { - "hubContractAddress": "0x209679fA3B658Cd0fC74473aF28243bfe78a9b12", - "evmOperationalWalletPrivateKey": "0x8ab3477bf3a1e0af66ab468fafd6cf982df99a59fee405d99861e7faf4db1f7b", - "evmManagementWalletPublicKey": "0xBaF76aC0d0ef9a2FFF76884d54C9D3e270290a43", - "evmOperationalWalletPublicKey": "0xBCc7F04c73214D160AA6C892FcA6DB881fb3E0F5" - } - } - } - }, - "network": { - "enabled": true, - "implementation": { - "libp2p-service": { - "package": "./network/implementation/libp2p-service.js", - "config": { - "kBucketSize": 20, - "connectionManager": { - "autoDial": true, - "autoDialInterval": 10e3, - "dialTimeout": 2e3 - }, - "peerRouting": { - "refreshManager": { - "enabled": true, - "interval": 6e5, - "bootDelay": 2e3 - } - }, - "port": 9001, - "bootstrap": [ - "/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" - ] - } - } - } - }, - "repository": { - "enabled": true, - "implementation": { - "sequelize-repository": { - "config": { - "database": "operationaldbnode0", - "password": "" - } - } - } - }, - "tripleStore": { - "enabled": true, - "defaultImplementation": "ot-graphdb", - "implementation": { - "ot-graphdb": { - "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", - "config": { - "repository": "origintrail-test-0" - } - } - } - }, - "validation": { - "enabled": true, - "implementation": { - "merkle-validation": { - "package": "./validation/implementation/merkle-validation.js", - "config": {} - } - } - }, - "httpClient": { - "enabled": true, - "implementation": { - "express-http-client": { - "package": "./http-client/implementation/express-http-client.js", - "config": { - "useSsl": false, - "port": 8901, - "sslKeyPath": "/root/certs/privkey.pem", - "sslCertificatePath": "/root/certs/fullchain.pem", - "rateLimiter": { - "timeWindowSeconds": 60, - "maxRequests": 10 - } - } - } - } - } - }, - "operationalDatabase": { - "databaseName": "operationaldbnode0" - }, - "graphDatabase": { - "name": "origintrail-test-0" - }, - "rpcPort": 8901, - "appDataPath": "data0", - "minimumAckResponses": { - "publish": 2, - "get": 1 - }, - "auth": { - "ipBasedAuthEnabled": false - } -} diff --git a/test/bdd/steps/hooks.mjs b/test/bdd/steps/hooks.mjs index e288bf4398..3d87d6715b 100644 --- a/test/bdd/steps/hooks.mjs +++ b/test/bdd/steps/hooks.mjs @@ -4,7 +4,8 @@ import slugify from 'slugify'; import fs from 'fs'; import mysql from 'mysql2'; import graphdb from 'graphdb'; -const {http,server} = graphdb; + +const { http, server } = graphdb; process.env.NODE_ENV = 'test'; diff --git a/test/bdd/steps/lib/local-blockchain.mjs b/test/bdd/steps/lib/local-blockchain.mjs index 9c1fea85ad..65bfb71c50 100644 --- a/test/bdd/steps/lib/local-blockchain.mjs +++ b/test/bdd/steps/lib/local-blockchain.mjs @@ -229,6 +229,12 @@ class LocalBlockchain { this.logger.info( `\t Profile contract address: \t\t\t\t${this.contracts.profile.instance._address}`, ); + this.logger.info( + `\t ContentAsset contract address: \t\t\t\t${this.contracts.contentAsset.instance._address}`, + ); + this.logger.info( + `\t ContentAsset Storage contract address: \t\t\t\t${this.contracts.contentAssetStorage.instance._address}`, + ); accept(); }); }); diff --git a/test/unit/sparlql-query-service.test.js b/test/unit/sparlql-query-service.test.js index c2596f5be3..e253849100 100644 --- a/test/unit/sparlql-query-service.test.js +++ b/test/unit/sparlql-query-service.test.js @@ -1,7 +1,7 @@ -/* eslint-disable */ -// import Blazegraph from '../../src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph'; -// import GraphDB from '../../src/modules/triple-store/implementation/ot-graphdb/ot-graphdb'; -// import Fuseki from '../../src/modules/triple-store/implementation/ot-fuseki/ot-fuseki'; +// /* eslint-disable */ +// import Blazegraph from '../../src/modules/triple-store/implementation/ot-blazegraph/ot-blazegraph.js'; +// import GraphDB from '../../src/modules/triple-store/implementation/ot-graphdb/ot-graphdb.js'; +// import Fuseki from '../../src/modules/triple-store/implementation/ot-fuseki/ot-fuseki.js'; // const { // describe, // it, @@ -9,21 +9,19 @@ // after, // } = require('mocha'); // import chai from 'chai'; -// + // const { // assert, // expect, // } = chai; -// -// import Sparql from '../../external/sparqlquery-service'; + // import Logger from '../../src/logger/logger'; // import fs from 'fs'; -// import GraphdbService from '../../external/graphdb-service'; -// + // let sparqlService = null; // let logger = null; // chai.use(require('chai-as-promised')); -// + // this.makeId = function (length) { // let result = ''; // const characters = 'abcdefghijklmnopqrstuvwxyz0123456789'; @@ -34,7 +32,7 @@ // } // return result; // }; -// + // describe('Sparql module', () => { // before('Initialize Logger', async () => { // logger = new Logger('trace', false); @@ -76,11 +74,11 @@ // expect(() => sparqlService.createLimitQuery({ limit: 'abc' })) // .to // .throw(Error); -// + // expect(() => sparqlService.createLimitQuery({ limit: Math.random() })) // .to // .throw(Error); -// + // expect(sparqlService.createLimitQuery({})) // .to // .equal(''); @@ -88,27 +86,27 @@ // // eslint-disable-next-line no-bitwise // const random = (Math.random() * (99999999 - 1 + 1)) << 0; // const negativeRandom = random * -1; -// + // expect(sparqlService.createLimitQuery({ limit: random })) // .to // .equal(`LIMIT ${random}`); -// + // expect(() => sparqlService.createLimitQuery({ limit: negativeRandom })) // .to // .throw(Error); // }); -// + // it('Check FindAssertionsByKeyword Errorhandling', async () => { // await expect(sparqlService.findAssertionsByKeyword('abc', { limit: 'aaaa' }, false)) // .to // .be // .rejectedWith(Error); -// + // await expect(sparqlService.findAssertionsByKeyword('abc', { limit: '90' }, 'test')) // .to // .be // .rejectedWith(Error); -// + // await expect(sparqlService.findAssertionsByKeyword('abc', { // limit: '90', // prefix: 'test', @@ -117,9 +115,9 @@ // .be // .rejectedWith(Error); // }); -// + // it('Check FindAssertionsByKeyword functionality', async () => { -// + // let id = this.makeId(65); // const addTriple = await sparqlService.insert(` schema:hasKeywords "${id}" `, `did:dkg:${id}`); // // This can also be mocked if necessary @@ -132,7 +130,7 @@ // .be // .not // .empty; -// + // const testTwo = await sparqlService.findAssertionsByKeyword(id, { // limit: 5, // prefix: false, @@ -145,39 +143,39 @@ // .empty; // }) // .timeout(600000); -// + // it('Check createFilterParameter', async () => { // expect(sparqlService.createFilterParameter('', '')) // .to // .equal(''); -// + // expect(sparqlService.createFilterParameter('\'', '')) // .to // .equal(''); -// + // expect(sparqlService.createFilterParameter('\'', sparqlService.filtertype.KEYWORD)) // .to // .equal('FILTER (lcase(?keyword) = \'\\\'\')'); -// + // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.KEYWORD)) // .to // .equal('FILTER (lcase(?keyword) = \'abcd\')'); -// + // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.KEYWORDPREFIX)) // .to // .equal('FILTER contains(lcase(?keyword),\'abcd\')'); -// + // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.TYPES)) // .to // .equal('FILTER (?type IN (abcd))'); -// + // expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.ISSUERS)) // .to // .equal('FILTER (?issuer IN (abcd))'); // }); // it('Check FindAssetsByKeyword functionality', async () => { // //Add new entry, so we can check if we find it really -// + // let id = this.makeId(65); // let triples = ` // schema:hasKeywords "${id}" . @@ -186,13 +184,13 @@ // schema:hasIssuer "${id}" . // schema:hasType "${id}" . // `; -// + // const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); // expect(addTriple) // .to // .be // .true; -// + // const testContains = await sparqlService.findAssetsByKeyword(id.substring(1, 20), { // limit: 5, // prefix: true, @@ -203,7 +201,7 @@ // .be // .not // .empty; -// + // const testExact = await sparqlService.findAssetsByKeyword(id, { // limit: 5, // prefix: true, @@ -229,10 +227,10 @@ // .timeout(600000); // it('Check insert functionality', async () => { // // This can also be mocked if necessary -// + // let id = this.makeId(65); // const test = await sparqlService.insert(` schema:hasKeywords "${id}" `, `did:dkg:${id}`); -// + // // eslint-disable-next-line no-unused-expressions // expect(test) // .to @@ -240,10 +238,10 @@ // .true; // }) // .timeout(600000); -// + // it('Check assertions By Asset functionality', async () => { // // This can also be mocked if necessary -// + // let id = this.makeId(65); // let triples = ` // schema:hasKeywords "${id}" . @@ -252,13 +250,13 @@ // schema:hasIssuer "${id}" . // schema:hasType "${id}" . // `; -// + // const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); // expect(addTriple) // .to // .be // .true; -// + // const testExact = await sparqlService.assertionsByAsset(id, { // limit: 5, // prefix: true, @@ -269,13 +267,13 @@ // .be // .not // .empty; -// + // }) // .timeout(600000); -// + // it('Check find Assertions functionality', async () => { // // This can also be mocked if necessary -// + // let id = this.makeId(65); // let triples = ` // schema:hasKeywords "${id}" . @@ -284,13 +282,13 @@ // schema:hasIssuer "${id}" . // schema:hasType "${id}" . // `; -// + // const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); // expect(addTriple) // .to // .be // .true; -// + // const testExact = await sparqlService.findAssertions(triples); // // eslint-disable-next-line no-unused-expressions // expect(testExact) @@ -298,7 +296,7 @@ // .be // .not // .empty; -// + // }) // .timeout(600000); // }); diff --git a/test/utilities/dkg-client-helper.mjs b/test/utilities/dkg-client-helper.mjs index 52818ae616..a9d556e238 100644 --- a/test/utilities/dkg-client-helper.mjs +++ b/test/utilities/dkg-client-helper.mjs @@ -12,8 +12,6 @@ class DkgClientHelper { async publish(data, wallet) { const options = { visibility: 'public', - triplesNumber: 3, - chunksNumber: 3, epochsNum: 5, maxNumberOfRetries: 5, hashFunctionId: 1, @@ -42,13 +40,6 @@ class DkgClientHelper { }); } - async search(resultType, query) { - return this.client._searchRequest({ - resultType, - query, - }); - } - async query(query) { return this.client._queryRequest({ query, diff --git a/test/utilities/steps-utils.mjs b/test/utilities/steps-utils.mjs index df62f0a980..7302b0277f 100644 --- a/test/utilities/steps-utils.mjs +++ b/test/utilities/steps-utils.mjs @@ -1,42 +1,48 @@ -import {fork} from "child_process"; -const otNodeProcessPath = './test/bdd/steps/lib/ot-node-process.mjs'; -class StepsUtils { +import { fork } from 'child_process'; +const otNodeProcessPath = './test/bdd/steps/lib/ot-node-process.mjs'; +class StepsUtils { forkNode(nodeConfiguration) { - const forkedNode = fork(otNodeProcessPath, [], {silent: true}); + const forkedNode = fork(otNodeProcessPath, [], { silent: true }); forkedNode.send(JSON.stringify(nodeConfiguration)); return forkedNode; } - - createNodeConfiguration(blockchain, wallet, managementWallet, nodeIndex, nodeName, rpcPort, sharesTokenName, sharesTokenSymbol) { + createNodeConfiguration( + wallet, + managementWallet, + nodeIndex, + nodeName, + rpcPort, + networkPort, + sharesTokenName, + sharesTokenSymbol, + bootstrap = false, + ) { return { modules: { - blockchain: - { - defaultImplementation: 'ganache', - implementation: { - ganache: { - config: { - blockchainTitle: 'ganache', - networkId: 'ganache::testnet', - rpcEndpoints: ['http://localhost:7545'], - hubContractAddress: blockchain.getHubAddress(), - evmOperationalWalletPublicKey: wallet.address, - evmOperationalWalletPrivateKey: wallet.privateKey, - evmManagementWalletPublicKey: managementWallet.address, - evmManagementWalletPrivateKey: managementWallet.privateKey, - sharesTokenName, - sharesTokenSymbol - }, + blockchain: { + implementation: { + ganache: { + config: { + evmOperationalWalletPublicKey: wallet.address, + evmOperationalWalletPrivateKey: wallet.privateKey, + evmManagementWalletPublicKey: managementWallet.address, + evmManagementWalletPrivateKey: managementWallet.privateKey, + sharesTokenName, + sharesTokenSymbol, }, }, }, + }, network: { implementation: { 'libp2p-service': { config: { - port: 9001 + nodeIndex, + port: networkPort, + privateKey: bootstrap + ? 'CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ=' + : undefined, }, }, }, @@ -45,7 +51,9 @@ class StepsUtils { implementation: { 'sequelize-repository': { config: { - database: `operationaldbnode${nodeIndex}`, + database: bootstrap + ? 'operationaldbbootstrap' + : `operationaldbnode${nodeIndex}`, }, }, }, @@ -54,11 +62,45 @@ class StepsUtils { implementation: { 'ot-graphdb': { config: { - repository: nodeName, + repositories: { + "privateCurrent": { + "url": "http://localhost:7200", + "name": "private-current", + "username": "admin", + "password": "" + }, + "privateHistory": { + "url": "http://localhost:7200", + "name": "private-history", + "username": "admin", + "password": "" + }, + "publicCurrent": { + "url": "http://localhost:7200", + "name": "public-current", + "username": "admin", + "password": "" + }, + "publicHistory": { + "url": "http://localhost:7200", + "name": "public-history", + "username": "admin", + "password": "" + } + }, }, }, }, }, + validation: { + enabled: true, + implementation: { + 'merkle-validation': { + enabled: true, + package: './validation/implementation/merkle-validation.js', + }, + }, + }, httpClient: { implementation: { 'express-http-client': { @@ -69,14 +111,23 @@ class StepsUtils { }, }, }, + auth: { + ipBasedAuthEnabled: false, + }, operationalDatabase: { - databaseName: `operationaldbnode${nodeIndex}`, + databaseName: bootstrap + ? 'operationaldbbootstrap' + : `operationaldbnode${nodeIndex}`, }, rpcPort, - appDataPath: `test-data${nodeIndex}`, + appDataPath: bootstrap ? 'test-data-bootstrap' : `test-data${nodeIndex}`, graphDatabase: { name: nodeName, }, + minimumAckResponses: { + publish: 2, + get: 1, + }, }; } } diff --git a/tools/local-network-setup/.bootstrap_origintrail_noderc b/tools/local-network-setup/.bootstrap_origintrail_noderc deleted file mode 100644 index 5cc64e6a42..0000000000 --- a/tools/local-network-setup/.bootstrap_origintrail_noderc +++ /dev/null @@ -1,112 +0,0 @@ -{ - "logLevel": "trace", - "modules": { - "repository": { - "enabled": true, - "implementation": { - "sequelize-repository": { - "config": { - "database": "operationaldb0" - } - } - } - }, - "tripleStore": { - "enabled": true, - "defaultImplementation": "ot-graphdb", - "implementation": { - "ot-graphdb": { - "config": { - "url": "http://localhost:7200", - "repository": "repository0", - "username": "admin", - "password": "" - } - } - } - }, - "httpClient": { - "enabled": true, - "implementation": { - "express-http-client": { - "package": "./http-client/implementation/express-http-client.js", - "config": { - "port": 8900 - } - } - } - }, - "network": { - "enabled": true, - "implementation": { - "libp2p-service": { - "config": { - "port": 9100, - "privateKey": "CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ=", - "bootstrap": [] - } - } - } - }, - "blockchain": { - "defaultImplementation": "ganache", - "implementation": { - "ganache": { - "config": { - "blockchainTitle": "ganache", - "networkId": "ganache::testnet", - "rpcEndpoints": [ - "http://127.0.0.1:7545" - ], - "evmOperationalWalletPublicKey": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", - "evmOperationalWalletPrivateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e", - "evmManagementWalletPublicKey": "0xf68B2609F1E240e501D78c78276D7314ba298025", - "evmManagementWalletPrivateKey": "0x50409e18b20ba522c909a296b3c378af1c31fb458aa6478988c260b78956ab3d", - "sharesTokenName": "LocalNode0", - "sharesTokenSymbol": "LN0" - } - }, - "otp": { - "config": { - "evmOperationalWalletPublicKey": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", - "evmOperationalWalletPrivateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e", - "evmManagementWalletPublicKey": "0xBaF76aC0d0ef9a2FFF76884d54C9D3e270290a43" - } - }, - "polygon": { - "config": { - "rpcEndpoints": [ - "wss://polygon-mumbai.g.alchemy.com/v2/bkYqCNrIC0TX3nZHuwo_cx7xqP4pYQ9s", - "https://matic-mumbai.chainstacklabs.com", - "https://rpc-mumbai.matic.today", - "https://matic-testnet-archive-rpc.bwarelabs.com" - ], - "evmOperationalWalletPublicKey": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", - "evmOperationalWalletPrivateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e", - "evmManagementWalletPublicKey": "0xBaF76aC0d0ef9a2FFF76884d54C9D3e270290a43" - } - }, - "rinkeby": { - "package": "./blockchain/implementation/polygon/eth-service.js", - "config": { - "networkId": "eth::rinkeby", - "gasPriceOracleLink": "", - "rpcEndpoints": [ - "http://127.0.0.1:7545" - ], - "evmOperationalWalletPublicKey": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", - "evmOperationalWalletPrivateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e", - "evmManagementWalletPublicKey": "0xBaF76aC0d0ef9a2FFF76884d54C9D3e270290a43" - } - } - } - } - }, - "auth": { - "ipWhitelist": [ - "::1", - "127.0.0.1" - ] - }, - "appDataPath": "data0" -} \ No newline at end of file diff --git a/tools/local-network-setup/.dh_origintrail_noderc b/tools/local-network-setup/.dh_origintrail_noderc deleted file mode 100644 index 5f584b74f0..0000000000 --- a/tools/local-network-setup/.dh_origintrail_noderc +++ /dev/null @@ -1,88 +0,0 @@ -{ - "logLevel": "trace", - "modules": { - "httpClient": { - "enabled": true, - "implementation": { - "express-http-client": { - "package": "./http-client/implementation/express-http-client.js", - "config": {} - } - } - }, - "repository": { - "enabled": true, - "implementation": { - "sequelize-repository": { - "package": "./repository/implementation/sequelize/sequelize-repository.js", - "config": {} - } - } - }, - "tripleStore": { - "enabled": true, - "defaultImplementation": "ot-graphdb", - "implementation": { - "ot-graphdb": { - "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", - "config": {} - } - } - }, - "network": { - "enabled": true, - "implementation": { - "libp2p-service": { - "package": "./network/implementation/libp2p-service.js", - "config": { - "port": 9001, - "bootstrap": [ - "/ip4/0.0.0.0/tcp/9100/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" - ] - } - } - } - }, - "blockchain": { - "defaultImplementation": "ganache", - "implementation": { - "ganache": { - "package": "./blockchain/implementation/ganache/ganache-service.js", - "config": { - "evmOperationalWalletPublicKey": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", - "rpcEndpoints": [], - "evmOperationalWalletPrivateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e" - } - }, - "otp": { - "config": { - "evmOperationalWalletPublicKey": "...", - "evmOperationalWalletPrivateKey": "...", - "rpcEndpoints": [], - "evmManagementWalletPublicKey": "..." - } - }, - "polygon": { - "config": { - "rpcEndpoints": [] - } - }, - "rinkeby": { - "package": "./blockchain/implementation/polygon/eth-service.js", - "config": { - "networkId": "eth::rinkeby", - "hubContractAddress": "", - "gasPriceOracleLink": "", - "rpcEndpoints": [] - } - } - } - } - }, - "auth": { - "ipWhitelist": [ - "::1", - "127.0.0.1" - ] - } -} diff --git a/tools/local-network-setup/.origintrail_noderc_template.json b/tools/local-network-setup/.origintrail_noderc_template.json new file mode 100644 index 0000000000..267834e7cc --- /dev/null +++ b/tools/local-network-setup/.origintrail_noderc_template.json @@ -0,0 +1,179 @@ +{ + "logLevel": "trace", + "modules": { + "httpClient": { + "enabled": true, + "implementation": { + "express-http-client": { + "package": "./http-client/implementation/express-http-client.js", + "config": {} + } + } + }, + "repository": { + "enabled": true, + "implementation": { + "sequelize-repository": { + "package": "./repository/implementation/sequelize/sequelize-repository.js", + "config": {} + } + } + }, + "tripleStore": { + "enabled": true, + "implementation": { + "ot-blazegraph": { + "enabled": false, + "package": "./triple-store/implementation/ot-blazegraph/ot-blazegraph.js", + "config": { + "repositories": { + "privateCurrent": { + "url": "http://localhost:9999", + "name": "private-current", + "username": "admin", + "password": "" + }, + "privateHistory": { + "url": "http://localhost:9999", + "name": "private-history", + "username": "admin", + "password": "" + }, + "publicCurrent": { + "url": "http://localhost:9999", + "name": "public-current", + "username": "admin", + "password": "" + }, + "publicHistory": { + "url": "http://localhost:9999", + "name": "public-history", + "username": "admin", + "password": "" + } + } + } + }, + "ot-fuseki": { + "enabled": false, + "package": "./triple-store/implementation/ot-fuseki/ot-fuseki.js", + "config": { + "repositories": { + "privateCurrent": { + "url": "http://localhost:3030", + "name": "private-current", + "username": "admin", + "password": "" + }, + "privateHistory": { + "url": "http://localhost:3030", + "name": "private-history", + "username": "admin", + "password": "" + }, + "publicCurrent": { + "url": "http://localhost:3030", + "name": "public-current", + "username": "admin", + "password": "" + }, + "publicHistory": { + "url": "http://localhost:3030", + "name": "public-history", + "username": "admin", + "password": "" + } + } + } + }, + "ot-graphdb": { + "enabled": false, + "package": "./triple-store/implementation/ot-graphdb/ot-graphdb.js", + "config": { + "repositories": { + "privateCurrent": { + "url": "http://localhost:7200", + "name": "private-current", + "username": "admin", + "password": "" + }, + "privateHistory": { + "url": "http://localhost:7200", + "name": "private-history", + "username": "admin", + "password": "" + }, + "publicCurrent": { + "url": "http://localhost:7200", + "name": "public-current", + "username": "admin", + "password": "" + }, + "publicHistory": { + "url": "http://localhost:7200", + "name": "public-history", + "username": "admin", + "password": "" + } + } + } + } + } + }, + "network": { + "enabled": true, + "implementation": { + "libp2p-service": { + "package": "./network/implementation/libp2p-service.js", + "config": { + "port": 9001, + "bootstrap": [ + "/ip4/0.0.0.0/tcp/9100/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj" + ] + } + } + } + }, + "blockchain": { + "defaultImplementation": "ganache", + "implementation": { + "ganache": { + "package": "./blockchain/implementation/ganache/ganache-service.js", + "config": { + "evmOperationalWalletPublicKey": "0xd6879C0A03aDD8cFc43825A42a3F3CF44DB7D2b9", + "rpcEndpoints": [], + "evmOperationalWalletPrivateKey": "0x02b39cac1532bef9dba3e36ec32d3de1e9a88f1dda597d3ac6e2130aed9adc4e" + } + }, + "otp": { + "config": { + "evmOperationalWalletPublicKey": "...", + "evmOperationalWalletPrivateKey": "...", + "rpcEndpoints": [], + "evmManagementWalletPublicKey": "..." + } + }, + "polygon": { + "config": { + "rpcEndpoints": [] + } + }, + "rinkeby": { + "package": "./blockchain/implementation/polygon/eth-service.js", + "config": { + "networkId": "eth::rinkeby", + "hubContractAddress": "", + "gasPriceOracleLink": "", + "rpcEndpoints": [] + } + } + } + } + }, + "auth": { + "ipWhitelist": [ + "::1", + "127.0.0.1" + ] + } +} diff --git a/tools/local-network-setup/generate-config-files.js b/tools/local-network-setup/generate-config-files.js index 05783f9050..5ccd5d48b6 100644 --- a/tools/local-network-setup/generate-config-files.js +++ b/tools/local-network-setup/generate-config-files.js @@ -2,94 +2,148 @@ import 'dotenv/config'; import mysql from 'mysql2'; import path from 'path'; -import fs from 'fs'; -import graphdb from 'graphdb'; -import appRootPath from 'app-root-path'; -import { LIBP2P_KEY_DIRECTORY, LIBP2P_KEY_FILENAME } from '../../src/constants/constants.js'; +import fs from 'fs-extra'; +import TripleStoreModuleManager from '../../src/modules/triple-store/triple-store-module-manager.js'; +import Logger from '../../src/logger/logger.js'; -const { server, http } = graphdb; +const { readFile, writeFile, stat } = fs; + +const generalConfig = JSON.parse(await readFile('./config/config.json')); +const templatePath = path.join('./tools/local-network-setup/.origintrail_noderc_template.json'); +const keys = JSON.parse(await readFile('./tools/local-network-setup/keys.json')); + +const logger = new Logger(generalConfig.development.logLevel); const numberOfNodes = parseInt(process.argv[2], 10); -const network = process.argv[3]; -const hubContractAddress = process.argv[4]; +const blockchain = process.argv[3]; +const tripleStoreImplementation = process.argv[4]; +const hubContractAddress = process.argv[5]; +const libp2pBootstrapPrivateKey = + 'CAAS4QQwggJdAgEAAoGBALOYSCZsmINMpFdH8ydA9CL46fB08F3ELfb9qiIq+z4RhsFwi7lByysRnYT/NLm8jZ4RvlsSqOn2ZORJwBywYD5MCvU1TbEWGKxl5LriW85ZGepUwiTZJgZdDmoLIawkpSdmUOc1Fbnflhmj/XzAxlnl30yaa/YvKgnWtZI1/IwfAgMBAAECgYEAiZq2PWqbeI6ypIVmUr87z8f0Rt7yhIWZylMVllRkaGw5WeGHzQwSRQ+cJ5j6pw1HXMOvnEwxzAGT0C6J2fFx60C6R90TPos9W0zSU+XXLHA7AtazjlSnp6vHD+RxcoUhm1RUPeKU6OuUNcQVJu1ZOx6cAcP/I8cqL38JUOOS7XECQQDex9WUKtDnpHEHU/fl7SvCt0y2FbGgGdhq6k8nrWtBladP5SoRUFuQhCY8a20fszyiAIfxQrtpQw1iFPBpzoq1AkEAzl/s3XPGi5vFSNGLsLqbVKbvoW9RUaGN8o4rU9oZmPFL31Jo9FLA744YRer6dYE7jJMel7h9VVWsqa9oLGS8AwJALYwfv45Nbb6yGTRyr4Cg/MtrFKM00K3YEGvdSRhsoFkPfwc0ZZvPTKmoA5xXEC8eC2UeZhYlqOy7lL0BNjCzLQJBAMpvcgtwa8u6SvU5B0ueYIvTDLBQX3YxgOny5zFjeUR7PS+cyPMQ0cyql8jNzEzDLcSg85tkDx1L4wi31Pnm/j0CQFH/6MYn3r9benPm2bYSe9aoJp7y6ht2DmXmoveNbjlEbb8f7jAvYoTklJxmJCcrdbNx/iCj2BuAinPPgEmUzfQ='; + +logger.info(`Generating config for ${numberOfNodes} node(s)`); -const dhTemplatePath = './tools/local-network-setup/.dh_origintrail_noderc'; -const bootstrapTemplatePath = './tools/local-network-setup/.bootstrap_origintrail_noderc'; +/******************************** + CONFIG GENERATION +*********************************/ + +const promises = []; +for (let i = 0; i < numberOfNodes; i += 1) { + promises.push(generateNodeConfig(i)); +} +await Promise.all(promises); -const generalConfig = JSON.parse(fs.readFileSync('./config/config.json')); -const dhTemplate = JSON.parse(fs.readFileSync(dhTemplatePath)); -const bootstrapTemplate = JSON.parse(fs.readFileSync(bootstrapTemplatePath)); -const keys = JSON.parse(fs.readFileSync('./tools/local-network-setup/keys.json')); +/******************************** + FUNCTIONS DEFINITIONS +*********************************/ -console.log('Preparing keys for blockchain'); +async function generateNodeConfig(nodeIndex) { + const configPath = path.join( + `./tools/local-network-setup/.node${nodeIndex}_origintrail_noderc.json`, + ); -if (!keys) { - console.log('Missing blockchain keys'); - process.exit(1); + if (!(await fileExists(configPath))) { + const template = JSON.parse(await readFile(templatePath)); + + logger.info(`Configuring node ${nodeIndex}`); + template.modules.tripleStore = generateTripleStoreConfig( + template.modules.tripleStore, + nodeIndex, + ); + template.modules.blockchain = generateBlockchainConfig( + template.modules.blockchain, + nodeIndex, + ); + template.modules.httpClient = generateHttpClientConfig( + template.modules.httpClient, + nodeIndex, + ); + template.modules.network = generateNetworkConfig(template.modules.network, nodeIndex); + template.modules.repository = generateRepositoryConfig( + template.modules.repository, + nodeIndex, + ); + template.appDataPath = `data${nodeIndex}`; + template.logLevel = process.env.LOG_LEVEL ?? template.logLevel; + + await writeFile(configPath, JSON.stringify(template, null, 4)); + } + const config = JSON.parse(await readFile(configPath)); + await dropDatabase( + `operationaldb${nodeIndex}`, + generalConfig.development.modules.repository.implementation['sequelize-repository'].config, + ); + await deleteTripleStoreRepositories(config); } -console.log(`Generating ${numberOfNodes} total nodes`); +function generateTripleStoreConfig(templateTripleStoreConfig, nodeIndex) { + const tripleStoreConfig = JSON.parse(JSON.stringify(templateTripleStoreConfig)); + + for (const implementationName in tripleStoreConfig.implementation) { + for (const [repository, config] of Object.entries( + tripleStoreConfig.implementation[implementationName].config.repositories, + )) { + tripleStoreConfig.implementation[implementationName].config.repositories[ + repository + ].name = `${config.name}-${nodeIndex}`; + } + tripleStoreConfig.implementation[implementationName].enabled = + implementationName === tripleStoreImplementation ? true : false; + } -for (let i = 0; i < numberOfNodes; i += 1) { - const tripleStoreConfig = { - ...generalConfig.development.modules.tripleStore.implementation['ot-graphdb'].config, - repository: `repository${i}`, - }; - const blockchainConfig = { + return tripleStoreConfig; +} + +function generateBlockchainConfig(templateBlockchainConfig, nodeIndex) { + const blockchainConfig = JSON.parse(JSON.stringify(templateBlockchainConfig)); + + blockchainConfig.implementation[blockchain].config = { + ...blockchainConfig.implementation[blockchain].config, hubContractAddress, rpcEndpoints: [process.env.RPC_ENDPOINT], - evmOperationalWalletPublicKey: keys.publicKey[i], - evmOperationalWalletPrivateKey: keys.privateKey[i], - evmManagementWalletPublicKey: keys.publicKey[keys.publicKey.length - 1 - i], - evmManagementWalletPrivateKey: keys.privateKey[keys.privateKey.length - 1 - i], - sharesTokenName: `LocalNode${i}`, - sharesTokenSymbol: `LN${i}`, + evmOperationalWalletPublicKey: keys.publicKey[nodeIndex], + evmOperationalWalletPrivateKey: keys.privateKey[nodeIndex], + evmManagementWalletPublicKey: keys.publicKey[keys.publicKey.length - 1 - nodeIndex], + evmManagementWalletPrivateKey: keys.privateKey[keys.privateKey.length - 1 - nodeIndex], + sharesTokenName: `LocalNode${nodeIndex}`, + sharesTokenSymbol: `LN${nodeIndex}`, }; - let appDataPath = `data${i}`; - let nodeName; - let template; - let templatePath; - if (i === 0) { - template = bootstrapTemplate; - templatePath = bootstrapTemplatePath; - nodeName = 'bootstrap'; - } else { - template = dhTemplate; - templatePath = path.join(`./tools/local-network-setup/.dh${i}_origintrail_noderc`); - nodeName = `DH${i}`; - } - template = JSON.parse(JSON.stringify(template)); - template.modules.blockchain.defaultImplementation = network; - template.modules.blockchain.implementation[network].config = { - ...template.modules.blockchain.implementation[network].config, - ...blockchainConfig, - }; + return blockchainConfig; +} - template.modules.httpClient.implementation['express-http-client'].config.port = 8900 + i; - template.modules.network.implementation['libp2p-service'].config.port = 9100 + i; - template.modules.repository.implementation[ - 'sequelize-repository' - ].config.database = `operationaldb${i}`; - template.modules.tripleStore.implementation['ot-graphdb'].config = tripleStoreConfig; - template.appDataPath = appDataPath; +function generateHttpClientConfig(templateHttpClientConfig, nodeIndex) { + const httpClientConfig = JSON.parse(JSON.stringify(templateHttpClientConfig)); + + httpClientConfig.implementation['express-http-client'].config.port = 8900 + nodeIndex; + + return httpClientConfig; +} - if (process.env.LOG_LEVEL) { - template.logLevel = process.env.LOG_LEVEL; +function generateNetworkConfig(templateNetworkConfig, nodeIndex) { + const networkConfig = JSON.parse(JSON.stringify(templateNetworkConfig)); + + networkConfig.implementation['libp2p-service'].config.port = 9100 + nodeIndex; + if (nodeIndex == 0) { + networkConfig.implementation['libp2p-service'].config.privateKey = + libp2pBootstrapPrivateKey; } - await dropDatabase( - `operationaldb${i}`, - generalConfig.development.modules.repository.implementation['sequelize-repository'].config, - ); - await deleteTripleStoreRepository(tripleStoreConfig); - console.log(`Configuring node ${nodeName}`); + return networkConfig; +} - fs.writeFileSync(templatePath, JSON.stringify(template, null, 2)); +function generateRepositoryConfig(templateRepositoryConfig, nodeIndex) { + const repositoryConfig = JSON.parse(JSON.stringify(templateRepositoryConfig)); + + repositoryConfig.implementation[ + 'sequelize-repository' + ].config.database = `operationaldb${nodeIndex}`; + + return repositoryConfig; } async function dropDatabase(name, config) { - console.log(`Dropping database: ${name}`); + logger.info(`Dropping database: ${name}`); const password = process.env.REPOSITORY_PASSWORD ?? config.password; const connection = mysql.createConnection({ database: name, @@ -100,20 +154,30 @@ async function dropDatabase(name, config) { try { await connection.promise().query(`DROP DATABASE IF EXISTS ${name};`); } catch (e) { - console.log(`Error while dropping database. Error: ${e}`); + logger.warn(`Error while dropping database. Error: ${e.message}`); } connection.destroy(); } -async function deleteTripleStoreRepository(config) { - console.log(`Deleting triple store: ${config.repository}`); - - const serverConfig = new server.ServerClientConfig(config.url) - .setTimeout(40000) - .setHeaders({ - Accept: http.RDFMimeType.N_QUADS, - }) - .setKeepAlive(true); - const s = new server.GraphDBServerClient(serverConfig); - s.deleteRepository(config.repository); +async function deleteTripleStoreRepositories(config) { + const tripleStoreModuleManager = new TripleStoreModuleManager({ config, logger }); + await tripleStoreModuleManager.initialize(); + + for (const implementationName of tripleStoreModuleManager.getImplementationNames()) { + const { module, config } = tripleStoreModuleManager.getImplementation(implementationName); + await Promise.all( + Object.keys(config.repositories).map((repository) => + module.deleteRepository(repository), + ), + ); + } +} + +async function fileExists(filePath) { + try { + await stat(filePath); + return true; + } catch (e) { + return false; + } } diff --git a/tools/local-network-setup/setup-macos-environment.sh b/tools/local-network-setup/setup-macos-environment.sh index 10ddd895c4..d926628e3d 100755 --- a/tools/local-network-setup/setup-macos-environment.sh +++ b/tools/local-network-setup/setup-macos-environment.sh @@ -2,6 +2,7 @@ pathToOtNode=$(pwd) numberOfNodes=4 network="ganache" +tripleStore="ot-graphdb" availableNetworks=("ganache" "rinkeby") export $(xargs < $pathToOtNode/.env) export ACCESS_KEY=$RPC_ENDPOINT @@ -31,6 +32,9 @@ while [ $# -gt 0 ]; do exit 1 fi ;; + --tripleStore=*) + tripleStore="${1#*=}" + ;; *) printf "***************************\n" printf "* Error: Invalid argument.*\n" @@ -66,7 +70,7 @@ echo ================================ echo ====== Generating configs ====== echo ================================ -node $pathToOtNode/tools/local-network-setup/generate-config-files.js $numberOfNodes $network $hubContractAddress +node $pathToOtNode/tools/local-network-setup/generate-config-files.js $numberOfNodes $network $tripleStore $hubContractAddress echo ================================ echo ======== Starting nodes ======== @@ -76,19 +80,13 @@ startNode() { echo Starting node $1 osascript -e "tell app \"Terminal\" do script \"cd $pathToOtNode - node index.js ./tools/local-network-setup/.$1_origintrail_noderc\" + node index.js ./tools/local-network-setup/.node$1_origintrail_noderc.json\" end tell" } -startNode bootstrap - -# Start only DC node and exit -if [[ $numberOfNodes -ne 1 ]] -then - i=1 - while [[ $i -lt $numberOfNodes ]] - do - startNode dh$i - ((i = i + 1)) - done -fi +i=0 +while [[ $i -lt $numberOfNodes ]] +do + startNode $i + ((i = i + 1)) +done From 76dfd81000378747395573940c5fec14d322b354 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:13:13 +0100 Subject: [PATCH 3/7] remove repository from local get function (#2340) --- .../protocols/publish/receiver/calculate-proofs-command.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/commands/protocols/publish/receiver/calculate-proofs-command.js b/src/commands/protocols/publish/receiver/calculate-proofs-command.js index bb33fd3048..b68a2e91a1 100644 --- a/src/commands/protocols/publish/receiver/calculate-proofs-command.js +++ b/src/commands/protocols/publish/receiver/calculate-proofs-command.js @@ -2,7 +2,6 @@ import EpochCommand from '../../common/epoch-command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, - TRIPLE_STORE_REPOSITORIES, COMMAND_RETRIES, } from '../../../../constants/constants.js'; @@ -69,11 +68,7 @@ class CalculateProofsCommand extends EpochCommand { epoch, ); - const nquads = await this.tripleStoreService.localGet( - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - assertionId, - operationId, - ); + const nquads = await this.tripleStoreService.localGet(assertionId, operationId); const { leaf, proof } = this.validationModuleManager.getMerkleProof( nquads, From 36c9786d84075dee6c1e70905ca4a66e70452e56 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:29:59 +0100 Subject: [PATCH 4/7] bump version (#2342) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6b8897dbf8..3b5107d498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.3", + "version": "6.0.3+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.3", + "version": "6.0.3+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index dfa506fb03..44c10e42a6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.3", + "version": "6.0.3+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", From b1d5b9008775f310c7984862b951e144efcc617b Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 23 Jan 2023 11:50:35 +0100 Subject: [PATCH 5/7] bump version to 6.0.3 (#2343) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3b5107d498..6b8897dbf8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.3+hotfix.1", + "version": "6.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.3+hotfix.1", + "version": "6.0.3", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 44c10e42a6..dfa506fb03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.3+hotfix.1", + "version": "6.0.3", "description": "OTNode V6", "main": "index.js", "type": "module", From a0569ad313ce6b6d6209fd4509bb494bc0c298a0 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 24 Jan 2023 15:40:58 +0100 Subject: [PATCH 6/7] handle publish local store requests (#2349) * add local store support * bump version --- package-lock.json | 4 +-- package.json | 2 +- .../local-store/local-store-command.js | 13 +++++-- .../http-api/publish-http-api-controller.js | 34 +++++++++++++------ 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6b8897dbf8..3b5107d498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.3", + "version": "6.0.3+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.3", + "version": "6.0.3+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index dfa506fb03..44c10e42a6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.3", + "version": "6.0.3+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/local-store/local-store-command.js b/src/commands/local-store/local-store-command.js index 536cd45632..cb113b5f54 100644 --- a/src/commands/local-store/local-store-command.js +++ b/src/commands/local-store/local-store-command.js @@ -5,6 +5,7 @@ class LocalStoreCommand extends Command { constructor(ctx) { super(ctx); this.tripleStoreService = ctx.tripleStoreService; + this.operationIdService = ctx.operationIdService; this.errorType = ERROR_TYPE.LOCAL_STORE.LOCAL_STORE_ERROR; } @@ -12,13 +13,14 @@ class LocalStoreCommand extends Command { async execute(command) { const { operationId } = command.data; + let assertions = []; try { await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.LOCAL_STORE.LOCAL_STORE_START, ); - const assertions = await this.operationIdService.getCachedOperationIdData(operationId); + assertions = await this.operationIdService.getCachedOperationIdData(operationId); await Promise.all( assertions.map(({ assertionId, assertion }) => @@ -43,9 +45,16 @@ class LocalStoreCommand extends Command { ); } catch (e) { await this.handleError(operationId, e.message, this.errorType, true); + return Command.empty(); } - return Command.empty(); + if (command?.sequence?.length) { + await this.operationIdService.cacheOperationIdData(operationId, { + assertion: assertions[0].assertion, + }); + } + + return this.continueSequence(command.data, command.sequence); } /** diff --git a/src/controllers/http-api/publish-http-api-controller.js b/src/controllers/http-api/publish-http-api-controller.js index c6f399efb2..d60f074d6e 100644 --- a/src/controllers/http-api/publish-http-api-controller.js +++ b/src/controllers/http-api/publish-http-api-controller.js @@ -31,22 +31,36 @@ class PublishController extends BaseController { const { assertion, assertionId, blockchain, contract, tokenId, hashFunctionId } = req.body; try { - await Promise.all([ - this.repositoryModuleManager.createOperationRecord( - this.operationService.getOperationName(), - operationId, - OPERATION_STATUS.IN_PROGRESS, - ), - this.operationIdService.cacheOperationIdData(operationId, { assertion }), - ]); + await this.repositoryModuleManager.createOperationRecord( + this.operationService.getOperationName(), + operationId, + OPERATION_STATUS.IN_PROGRESS, + ); this.logger.info( `Received asset with assertion id: ${assertionId}, blockchain: ${blockchain}, hub contract: ${contract}, token id: ${tokenId}`, ); + let commandSequence = []; + + if (req.body.localStore) { + commandSequence.push('localStoreCommand'); + await this.operationIdService.cacheOperationIdData(operationId, [ + { assertion, assertionId }, + ]); + } else { + await this.operationIdService.cacheOperationIdData(operationId, { assertion }); + } + + commandSequence = [ + ...commandSequence, + 'validateAssertionCommand', + 'networkPublishCommand', + ]; + await this.commandExecutor.add({ - name: 'validateAssertionCommand', - sequence: ['networkPublishCommand'], + name: commandSequence[0], + sequence: commandSequence.slice(1), delay: 0, period: 5000, retries: 3, From 8be66c79b7281cc06b68e95612f76a73ed19deec Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 26 Jan 2023 09:36:29 +0100 Subject: [PATCH 7/7] add copy assertions migration (#2351) --- scripts/copy-assertions.js | 154 +++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 scripts/copy-assertions.js diff --git a/scripts/copy-assertions.js b/scripts/copy-assertions.js new file mode 100644 index 0000000000..a107f6a88b --- /dev/null +++ b/scripts/copy-assertions.js @@ -0,0 +1,154 @@ +import 'dotenv/config'; +import fs from 'fs-extra'; +import rc from 'rc'; +import appRootPath from 'app-root-path'; +import path from 'path'; +import { TRIPLE_STORE_REPOSITORIES, SCHEMA_CONTEXT } from '../src/constants/constants.js'; +import TripleStoreModuleManager from '../src/modules/triple-store/triple-store-module-manager.js'; +import DataService from '../src/service/data-service.js'; +import Logger from '../src/logger/logger.js'; + +const { readFile } = fs; +const generalConfig = JSON.parse(await readFile(path.join(appRootPath.path, 'config/config.json'))); +const pjson = JSON.parse(await readFile(path.join(appRootPath.path, 'package.json'))); + +const defaultConfig = generalConfig[process.env.NODE_ENV]; + +const config = rc(pjson.name, defaultConfig); +const logger = new Logger(config.loglevel); + +const tripleStoreModuleManager = new TripleStoreModuleManager({ config, logger }); +await tripleStoreModuleManager.initialize(); +const dataService = new DataService({ config, logger }); + +const repositoryImplementations = {}; +for (const implementationName of tripleStoreModuleManager.getImplementationNames()) { + for (const repository in tripleStoreModuleManager.getImplementation(implementationName).module + .repositories) { + repositoryImplementations[repository] = implementationName; + } +} + +const fromRepository = TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT; +const fromImplementation = repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT]; +const fromRepositoryName = + tripleStoreModuleManager.getImplementation(fromImplementation).module.repositories[ + fromRepository + ].name; + +const toRepository = TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT; +const toImplementation = repositoryImplementations[TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT]; +const toRepositoryName = + tripleStoreModuleManager.getImplementation(toImplementation).module.repositories[toRepository] + .name; + +async function getAssertions(implementation, repository) { + const graphs = await tripleStoreModuleManager.select( + implementation, + repository, + `SELECT DISTINCT ?g + WHERE { + GRAPH ?g { ?s ?p ?o } + }`, + ); + + return (graphs ?? []).filter(({ g }) => g.startsWith('assertion:')).map(({ g }) => g); +} + +function logPercentage(index, max) { + const previousPercentage = (Math.max(0, index - 1) / max) * 100; + const currentPercentage = (index / max) * 100; + + if (Math.floor(currentPercentage) - Math.floor(previousPercentage) < 1) return; + + logger.debug(`Migration at ${Math.floor(currentPercentage * 10) / 10}%`); +} + +let toRepositoryAssertions = await getAssertions(toImplementation, toRepository); +logger.info( + `${toRepositoryAssertions.length} assertions found in ${toRepository} repository before migration`, +); + +logger.info( + `Starting to copy assertions from ${fromImplementation} repository ${fromRepository} with name ${fromRepositoryName} to repository ${toImplementation} repository ${toRepository} with name ${toRepositoryName}`, +); + +const fromRepositoryAssertions = await getAssertions(fromImplementation, fromRepository); +logger.info(`${fromRepositoryAssertions.length} assertions found in ${fromRepository}`); + +let completed = 0; +const copyAssertion = async (g) => { + if (!toRepositoryAssertions.includes(g)) { + let nquads; + try { + nquads = await tripleStoreModuleManager.construct( + fromImplementation, + fromRepository, + `PREFIX schema: <${SCHEMA_CONTEXT}> + CONSTRUCT { ?s ?p ?o } + WHERE { + { + GRAPH <${g}> + { + ?s ?p ?o . + } + } + }`, + ); + + nquads = await dataService.toNQuads(nquads, 'application/n-quads'); + } catch (error) { + logger.error( + `Error while getting assertion ${g.substring( + 'assertion:'.length, + )} from ${fromImplementation} repository ${fromRepository} with name ${fromRepositoryName}. Error: ${ + error.message + }`, + ); + process.exit(1); + } + + try { + await tripleStoreModuleManager.insertAssertion( + toImplementation, + toRepository, + g.substring('assertion:'.length), + nquads.join('\n'), + ); + } catch (error) { + logger.error( + `Error while inserting assertion ${g.substring( + 'assertion:'.length, + )} with nquads: ${nquads} in ${toImplementation} repository ${toRepository} with name ${toRepositoryName}. Error: ${ + error.message + }`, + ); + process.exit(1); + } + } + + completed += 1; + logPercentage(completed, fromRepositoryAssertions.length); +}; + +const start = Date.now(); +const concurrency = 10; +let promises = []; +for (let i = 0; i < fromRepositoryAssertions.length; i += 1) { + promises.push(copyAssertion(fromRepositoryAssertions[i])); + if (promises.length > concurrency) { + // eslint-disable-next-line no-await-in-loop + await Promise.all(promises); + promises = []; + } +} +await Promise.all(promises); + +const end = Date.now(); + +logger.info(`Migration completed! Lasted ${(end - start) / 1000} seconds.`); + +toRepositoryAssertions = await getAssertions(toImplementation, toRepository); +logger.info( + `${toRepositoryAssertions.length} assertions found in ${toRepository} repository after migration`, +);