diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000000..9acedddf3e --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,41 @@ +name: setup + +runs: + using: composite + steps: + - name: Setup NodeJS + id: nodejs + uses: actions/setup-node@v3 + with: + node-version: 16.x + cache: npm + + - name: Cache node modules + id: cache-node-modules + uses: actions/cache@v3 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node-modules-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-modules-${{ hashFiles('**/package-lock.json') }} + ${{ runner.os }}-node-modules- + + - name: Cache Blazegraph + id: cache-blazegraph + uses: actions/cache@v3 + with: + path: blazegraph.jar + key: ${{ runner.os }}-blazegraph-${{ hashFiles('blazegraph.jar') }} + restore-keys: ${{ runner.os }}-blazegraph- + + - if: steps.cache-node-modules.outputs.cache-hit != 'true' + name: Install dependencies & compile contracts + shell: bash + run: | + npm ci + npm explore dkg-evm-module -- npm run compile + + - if: steps.cache-blazegraph.outputs.cache-hit != 'true' + name: Download Blazegraph + shell: bash + run: wget https://github.com/blazegraph/database/releases/latest/download/blazegraph.jar diff --git a/.github/workflows/CHECK-lint.yml b/.github/workflows/CHECK-lint.yml deleted file mode 100644 index a8fca282c7..0000000000 --- a/.github/workflows/CHECK-lint.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: CHECK-lint - -#todo this test should be execute when opening PR to prerelease/release branches -on: [pull_request] -env: - NODE_ENV: test - ARTIFACTS_DIR: artifacts - CUCUMBER_ARTIFACTS_DIR: artifacts/cucumber -jobs: - check-lint: - #todo think about locking the version - version should be the same as the one in official documentation - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [16.x] - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: mkdir -p $ARTIFACTS_DIR - - run: sudo chmod -R 777 $ARTIFACTS_DIR - - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - - run: npm run lint; - - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: my-artifact - path: /home/runner/work/ot-node/ot-node/artifacts diff --git a/.github/workflows/TEST-bdd.yml b/.github/workflows/TEST-bdd.yml deleted file mode 100644 index 7fb92f8fea..0000000000 --- a/.github/workflows/TEST-bdd.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: TEST-bdd - -#todo this test should be execute when opening PR to prerelease/release branches -on: [pull_request] -env: - NODE_ENV: test - ARTIFACTS_DIR: artifacts - CUCUMBER_ARTIFACTS_DIR: artifacts/cucumber - REPOSITORY_PASSWORD: password -jobs: - - test-bdd: - #todo think about locking the version - version should be the same as the one in official documentation - runs-on: ubuntu-latest - services: - mysql: - image: mysql:5.7 - env: - MYSQL_DATABASE: operationaldb - MYSQL_USER: node - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: password - ports: - - 3306:3306 - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 - strategy: - matrix: - node-version: [16.x] - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - name: Download Blazegraph - run: wget https://github.com/blazegraph/database/releases/latest/download/blazegraph.jar - - name: Cache Blazegraph - uses: actions/cache@v2 - with: - path: blazegraph.jar - key: ${{ runner.os }}-blazegraph-${{ hashFiles('blazegraph.jar') }} - restore-keys: ${{ runner.os }}-blazegraph- - - run: /usr/bin/java -Djava.awt.headless=true -jar blazegraph.jar & - - name: Cache node_modules - uses: actions/cache@v2 - with: - path: | - ~/.npm - ./node_modules - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - run: npm ci - - run: mkdir -p $ARTIFACTS_DIR - - run: sudo chmod -R 777 $ARTIFACTS_DIR - - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - - run: npm explore dkg-evm-module -- npm run compile; - - run: npm run test:bdd; - - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: my-artifact - path: /home/runner/work/ot-node/ot-node/artifacts diff --git a/.github/workflows/TEST-unit.yml b/.github/workflows/TEST-unit.yml deleted file mode 100644 index 9b437e88e4..0000000000 --- a/.github/workflows/TEST-unit.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: TEST-unit - -#todo this test should be execute when opening PR to prerelease/release branches -on: [pull_request] -env: - NODE_ENV: test - ARTIFACTS_DIR: artifacts - CUCUMBER_ARTIFACTS_DIR: artifacts/cucumber - JWT_SECRET: aTx13FzDG+85j9b5s2G7IBEc5SJNJZZLPLe7RF8hu1xKgRKj46YFRx/z7fJi7iF2NnL7SHcxTzq7TySuPKWkdg/AYKEMD2p1I++qPYFHqg8KQeLArGjCYiqtf43i1Fgtya8z9qJXyegogMz/jYori2BJ8v6b4K3GkAw3XxiO7VaaEYktOp8qsRDcN3b+bITMZqztDvZdWp4EnViGjoES7fRFhKm/d/2C8URnQyGm6xgTR3xTfAjy7+milGmoPA0KU0nu+GsZIhOfeVc9Z2nfxOK/1JQykpjeBhNDYTOr31yW/xdvoW0Kq0PZ6JmM+yezLoyQXcYjavZ+X7cXjbREQg== -jobs: - - test-unit: - #todo think about locking the version - version should be the same as the one in official documentation - runs-on: ubuntu-latest - services: -# mysql: -# image: mysql:5.7 -# env: -# MYSQL_DATABASE: operationaldb -# MYSQL_USER: node -# MYSQL_PASSWORD: password -# MYSQL_ROOT_PASSWORD: password -# ports: -# - 3306:3306 -# options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 - graphdb: - image: khaller/graphdb-free:latest - ports: - - 7200:7200 - strategy: - matrix: - node-version: [16.x] - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: mkdir -p $ARTIFACTS_DIR - - run: sudo chmod -R 777 $ARTIFACTS_DIR - - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - - run: npm run test:unit; - - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: my-artifact - path: /home/runner/work/ot-node/ot-node/artifacts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000000..64356fb2f4 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,77 @@ +name: checks + +on: + pull_request: + branches: + - v6/develop + +env: + REPOSITORY_PASSWORD: password + JWT_SECRET: aTx13FzDG+85j9b5s2G7IBEc5SJNJZZLPLe7RF8hu1xKgRKj46YFRx/z7fJi7iF2NnL7SHcxTzq7TySuPKWkdg/AYKEMD2p1I++qPYFHqg8KQeLArGjCYiqtf43i1Fgtya8z9qJXyegogMz/jYori2BJ8v6b4K3GkAw3XxiO7VaaEYktOp8qsRDcN3b+bITMZqztDvZdWp4EnViGjoES7fRFhKm/d/2C8URnQyGm6xgTR3xTfAjy7+milGmoPA0KU0nu+GsZIhOfeVc9Z2nfxOK/1JQykpjeBhNDYTOr31yW/xdvoW0Kq0PZ6JmM+yezLoyQXcYjavZ+X7cXjbREQg== + +concurrency: + group: checks-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup + + - name: Run linter + run: npm run lint + + unit-tests: + runs-on: ubuntu-latest + services: + graphdb: + image: khaller/graphdb-free:latest + ports: + - 7200:7200 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup + + - name: Run unit tests + run: npm run test:unit + + bdd-tests: + runs-on: ubuntu-latest + services: + mysql: + image: mysql:5.7 + env: + MYSQL_DATABASE: operationaldb + MYSQL_USER: node + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup + + - name: Run Blazegraph + run: /usr/bin/java -Djava.awt.headless=true -jar blazegraph.jar & + + - name: Run BDD tests + run: npm run test:bdd + + - name: Upload log files + if: '!cancelled()' + uses: actions/upload-artifact@v3 + with: + name: bdd-test-logs + path: ./test/bdd/log/ diff --git a/.github/workflows/release-drafter-config.yml b/.github/workflows/release-drafter-config.yml index 705fdc435a..069a2ed64a 100644 --- a/.github/workflows/release-drafter-config.yml +++ b/.github/workflows/release-drafter-config.yml @@ -1,8 +1,7 @@ -name: Release Drafter +name: release-drafter on: push: - # branches to consider in the event; optional, defaults to all branches: - develop diff --git a/.github/workflows/update-cache.yml b/.github/workflows/update-cache.yml new file mode 100644 index 0000000000..cbf1f24c68 --- /dev/null +++ b/.github/workflows/update-cache.yml @@ -0,0 +1,22 @@ +name: update-cache + +on: + push: + branches: + - v6/develop + +concurrency: + group: update-cache-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-cache: + name: Build Cache + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup diff --git a/ot-node.js b/ot-node.js index 17407c3894..9ac39cf525 100644 --- a/ot-node.js +++ b/ot-node.js @@ -17,6 +17,7 @@ import RemoveAgreementStartEndTimeMigration from './src/migration/remove-agreeme import MarkOldBlockchainEventsAsProcessedMigration from './src/migration/mark-old-blockchain-events-as-processed-migration.js'; import TripleStoreMetadataMigration from './src/migration/triple-store-metadata-migration.js'; import RemoveOldEpochCommandsMigration from './src/migration/remove-old-epoch-commands-migration.js'; +import PendingStorageMigration from './src/migration/pending-storage-migration.js'; const require = createRequire(import.meta.url); const pjson = require('./package.json'); @@ -58,6 +59,7 @@ class OTNode { await this.executeTripleStoreMetadataMigration(); await this.executeServiceAgreementsMetadataMigration(); await this.executeRemoveOldEpochCommandsMigration(); + await this.executePendingStorageMigration(); await this.createProfiles(); @@ -421,6 +423,23 @@ class OTNode { } } + async executePendingStorageMigration() { + if ( + process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || + process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST + ) + return; + + const migration = new PendingStorageMigration( + 'pendingStorageMigration', + this.logger, + this.config, + ); + if (!(await migration.migrationAlreadyExecuted())) { + await migration.migrate(); + } + } + async initializeShardingTableService() { try { const shardingTableService = this.container.resolve('shardingTableService'); diff --git a/package-lock.json b/package-lock.json index 611b263c66..7ad232345b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.10", + "version": "6.0.11", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.10", + "version": "6.0.11", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", @@ -2687,6 +2687,26 @@ "regexp-match-indices": "1.0.2" } }, + "node_modules/@cucumber/cucumber/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@cucumber/gherkin": { "version": "26.0.3", "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-26.0.3.tgz", @@ -4304,9 +4324,9 @@ } }, "node_modules/@openzeppelin/contracts": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.8.3.tgz", - "integrity": "sha512-bQHV8R9Me8IaJoJ2vPG4rXcL7seB7YVuskr4f+f5RyOStSZetwzkWtoqDMl5erkBJy0lDRUnIR2WIkPiC0GJlg==" + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.2.tgz", + "integrity": "sha512-mO+y6JaqXjWeMh9glYVzVu8HYPGknAAnWyxTRhGeckOruyXQMNnlcW6w/Dx9ftLeIQk6N+ZJFuVmTwF7lEIFrg==" }, "node_modules/@polkadot/api": { "version": "9.14.2", @@ -8283,9 +8303,9 @@ } }, "node_modules/dottie": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.3.tgz", - "integrity": "sha512-4liA0PuRkZWQFQjwBypdxPfZaRWiv5tkhMXY2hzsa2pNf5s7U3m9cwUchfNKe8wZQxdGPQQzO6Rm2uGe0rvohQ==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.4.tgz", + "integrity": "sha512-iz64WUOmp/ECQhWMJjTWFzJN/wQ7RJ5v/a6A2OiCwjaGCpNo66WGIjlSf+IULO9DQd0b4cFawLOTbiKSrpKodw==" }, "node_modules/duplexer2": { "version": "0.1.4", @@ -9984,6 +10004,25 @@ "rimraf": "^2.6.3" } }, + "node_modules/fs-jetpack/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fs-jetpack/node_modules/rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -10036,6 +10075,25 @@ "node": ">=0.6" } }, + "node_modules/fstream/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fstream/node_modules/rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -10181,25 +10239,6 @@ "assert-plus": "^1.0.0" } }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -14522,6 +14561,26 @@ "node": ">=8" } }, + "node_modules/nyc/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/nyc/node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -16258,6 +16317,25 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/ripemd160": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", @@ -16922,6 +17000,25 @@ "rimraf": "^2.2.8" } }, + "node_modules/solc/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/solc/node_modules/jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", @@ -17652,6 +17749,26 @@ "node": ">=8" } }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -21737,6 +21854,22 @@ "xmlbuilder": "^15.1.1", "yaml": "1.10.2", "yup": "^0.32.11" + }, + "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "@cucumber/cucumber-expressions": { @@ -22809,9 +22942,9 @@ "optional": true }, "@openzeppelin/contracts": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.8.3.tgz", - "integrity": "sha512-bQHV8R9Me8IaJoJ2vPG4rXcL7seB7YVuskr4f+f5RyOStSZetwzkWtoqDMl5erkBJy0lDRUnIR2WIkPiC0GJlg==" + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.2.tgz", + "integrity": "sha512-mO+y6JaqXjWeMh9glYVzVu8HYPGknAAnWyxTRhGeckOruyXQMNnlcW6w/Dx9ftLeIQk6N+ZJFuVmTwF7lEIFrg==" }, "@polkadot/api": { "version": "9.14.2", @@ -26004,9 +26137,9 @@ "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==" }, "dottie": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.3.tgz", - "integrity": "sha512-4liA0PuRkZWQFQjwBypdxPfZaRWiv5tkhMXY2hzsa2pNf5s7U3m9cwUchfNKe8wZQxdGPQQzO6Rm2uGe0rvohQ==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.4.tgz", + "integrity": "sha512-iz64WUOmp/ECQhWMJjTWFzJN/wQ7RJ5v/a6A2OiCwjaGCpNo66WGIjlSf+IULO9DQd0b4cFawLOTbiKSrpKodw==" }, "duplexer2": { "version": "0.1.4", @@ -27390,6 +27523,19 @@ "rimraf": "^2.6.3" }, "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -27431,6 +27577,19 @@ "rimraf": "2" }, "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -27539,19 +27698,6 @@ "assert-plus": "^1.0.0" } }, - "glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, "glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -30919,6 +31065,20 @@ "path-exists": "^4.0.0" } }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -32265,6 +32425,21 @@ "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "requires": { "glob": "^7.1.3" + }, + "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "ripemd160": { @@ -32767,6 +32942,19 @@ "rimraf": "^2.2.8" } }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", @@ -33342,6 +33530,22 @@ "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", "minimatch": "^3.0.4" + }, + "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "text-hex": { diff --git a/package.json b/package.json index c2d531f07f..aaeca8c038 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.10", + "version": "6.0.11", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/cleaners/blockchain-event-cleaner-command.js b/src/commands/cleaners/blockchain-event-cleaner-command.js new file mode 100644 index 0000000000..ea0017793a --- /dev/null +++ b/src/commands/cleaners/blockchain-event-cleaner-command.js @@ -0,0 +1,42 @@ +import { + PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY, + ARCHIVE_BLOCKCHAIN_EVENTS_FOLDER, +} from '../../constants/constants.js'; +import CleanerCommand from './cleaner-command.js'; + +class BlockchainEventCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedEvents( + nowTimestamp - PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_BLOCKCHAIN_EVENTS_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeEvents(ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'blockchainEventCleanerCommand', + data: {}, + period: PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default BlockchainEventCleanerCommand; diff --git a/src/commands/cleaners/cleaner-command.js b/src/commands/cleaners/cleaner-command.js new file mode 100644 index 0000000000..38dc0685f2 --- /dev/null +++ b/src/commands/cleaners/cleaner-command.js @@ -0,0 +1,75 @@ +import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER } from '../../constants/constants.js'; +import Command from '../command.js'; + +class CleanerCommand extends Command { + constructor(ctx) { + super(ctx); + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.archiveService = ctx.archiveService; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute() { + const nowTimestamp = Date.now(); + + let rowsForRemoval = await this.findRowsForRemoval(nowTimestamp); + + while (rowsForRemoval?.length >= REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER) { + const archiveName = this.getArchiveName(rowsForRemoval); + + // eslint-disable-next-line no-await-in-loop + await this.archiveService.archiveData( + this.getArchiveFolderName(), + archiveName, + rowsForRemoval, + ); + + // remove from database; + const ids = rowsForRemoval.map((command) => command.id); + // eslint-disable-next-line no-await-in-loop + await this.deleteRows(ids); + + // eslint-disable-next-line no-await-in-loop + rowsForRemoval = await this.findRowsForRemoval(nowTimestamp); + } + + return Command.repeat(); + } + + getArchiveName(rowsForRemoval) { + const firstTimestamp = new Date(rowsForRemoval[0].createdAt).getTime(); + const lastTimestamp = new Date( + rowsForRemoval[rowsForRemoval.length - 1].createdAt, + ).getTime(); + return `${firstTimestamp}-${lastTimestamp}.json`; + } + + // eslint-disable-next-line no-unused-vars + async findRowsForRemoval(nowTimestamp) { + throw Error('findRowsForRemoval not implemented'); + } + + getArchiveFolderName() { + throw Error('getArchiveFolderName not implemented'); + } + + // eslint-disable-next-line no-unused-vars + async deleteRows(ids) { + throw Error('deleteRows not implemented'); + } + + /** + * Recover system from failure + * @param command + * @param error + */ + async recover(command, error) { + this.logger.warn(`Failed to clean operational db data: error: ${error.message}`); + return Command.repeat(); + } +} + +export default CleanerCommand; diff --git a/src/commands/cleaners/commands-cleaner-command.js b/src/commands/cleaners/commands-cleaner-command.js new file mode 100644 index 0000000000..746d182331 --- /dev/null +++ b/src/commands/cleaners/commands-cleaner-command.js @@ -0,0 +1,41 @@ +import { + FINALIZED_COMMAND_CLEANUP_TIME_MILLS, + ARCHIVE_COMMANDS_FOLDER, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, +} from '../../constants/constants.js'; +import CleanerCommand from './cleaner-command.js'; + +class CommandsCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findFinalizedCommands( + nowTimestamp, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_COMMANDS_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeCommands(ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'commandsCleanerCommand', + data: {}, + period: FINALIZED_COMMAND_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default CommandsCleanerCommand; diff --git a/src/commands/cleaners/get-cleaner-command.js b/src/commands/cleaners/get-cleaner-command.js new file mode 100644 index 0000000000..3991623871 --- /dev/null +++ b/src/commands/cleaners/get-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + GET_CLEANUP_TIME_DELAY, + GET_CLEANUP_TIME_MILLS, + ARCHIVE_GET_FOLDER, +} from '../../constants/constants.js'; + +class GetCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperations( + OPERATIONS.GET, + nowTimestamp - GET_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_GET_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationRecords(OPERATIONS.GET, ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'getCleanerCommand', + data: {}, + period: GET_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default GetCleanerCommand; diff --git a/src/commands/cleaners/get-response-cleaner-command.js b/src/commands/cleaners/get-response-cleaner-command.js new file mode 100644 index 0000000000..d79e90ba02 --- /dev/null +++ b/src/commands/cleaners/get-response-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + GET_RESPONSE_CLEANUP_TIME_DELAY, + GET_RESPONSE_CLEANUP_TIME_MILLS, + ARCHIVE_GET_RESPONSES_FOLDER, +} from '../../constants/constants.js'; + +class GetResponseCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperationResponse( + nowTimestamp - GET_RESPONSE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS.GET, + ); + } + + getArchiveFolderName() { + return ARCHIVE_GET_RESPONSES_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationResponse(ids, OPERATIONS.GET); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'getResponseCleanerCommand', + data: {}, + period: GET_RESPONSE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default GetResponseCleanerCommand; diff --git a/src/commands/common/operation-id-cleaner-command.js b/src/commands/cleaners/operation-id-cleaner-command.js similarity index 95% rename from src/commands/common/operation-id-cleaner-command.js rename to src/commands/cleaners/operation-id-cleaner-command.js index 7f43a43826..d1816e7d4c 100644 --- a/src/commands/common/operation-id-cleaner-command.js +++ b/src/commands/cleaners/operation-id-cleaner-command.js @@ -1,6 +1,7 @@ import Command from '../command.js'; import { BYTES_IN_KILOBYTE, + OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, OPERATION_ID_STATUS, } from '../../constants/constants.js'; @@ -39,6 +40,7 @@ class OperationIdCleanerCommand extends Command { } removed = await this.operationIdService.removeExpiredOperationIdFileCache( OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, + OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, ); if (removed) { this.logger.debug(`Successfully removed ${removed} expired cached operation files`); diff --git a/src/commands/cleaners/publish-cleaner-command.js b/src/commands/cleaners/publish-cleaner-command.js new file mode 100644 index 0000000000..cfa93ea85e --- /dev/null +++ b/src/commands/cleaners/publish-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + PUBLISH_CLEANUP_TIME_DELAY, + PUBLISH_CLEANUP_TIME_MILLS, + ARCHIVE_PUBLISH_FOLDER, +} from '../../constants/constants.js'; + +class PublishCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperations( + OPERATIONS.PUBLISH, + nowTimestamp - PUBLISH_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_PUBLISH_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationRecords(OPERATIONS.PUBLISH, ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'publishCleanerCommand', + data: {}, + period: PUBLISH_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default PublishCleanerCommand; diff --git a/src/commands/cleaners/publish-response-cleaner-command.js b/src/commands/cleaners/publish-response-cleaner-command.js new file mode 100644 index 0000000000..85ff500af5 --- /dev/null +++ b/src/commands/cleaners/publish-response-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + PUBLISH_RESPONSE_CLEANUP_TIME_DELAY, + PUBLISH_RESPONSE_CLEANUP_TIME_MILLS, + ARCHIVE_PUBLISH_RESPONSES_FOLDER, +} from '../../constants/constants.js'; + +class PublishResponseCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperationResponse( + nowTimestamp - PUBLISH_RESPONSE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS.PUBLISH, + ); + } + + getArchiveFolderName() { + return ARCHIVE_PUBLISH_RESPONSES_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationResponse(ids, OPERATIONS.PUBLISH); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'publishResponseCleanerCommand', + data: {}, + period: PUBLISH_RESPONSE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default PublishResponseCleanerCommand; diff --git a/src/commands/cleaners/update-cleaner-command.js b/src/commands/cleaners/update-cleaner-command.js new file mode 100644 index 0000000000..9f57fae2e4 --- /dev/null +++ b/src/commands/cleaners/update-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + UPDATE_CLEANUP_TIME_DELAY, + UPDATE_CLEANUP_TIME_MILLS, + ARCHIVE_UPDATE_FOLDER, +} from '../../constants/constants.js'; + +class UpdateCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperations( + OPERATIONS.UPDATE, + nowTimestamp - UPDATE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_UPDATE_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationRecords(OPERATIONS.UPDATE, ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'publishCleanerCommand', + data: {}, + period: UPDATE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default UpdateCleanerCommand; diff --git a/src/commands/cleaners/update-response-cleaner-command.js b/src/commands/cleaners/update-response-cleaner-command.js new file mode 100644 index 0000000000..d5066e7263 --- /dev/null +++ b/src/commands/cleaners/update-response-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + UPDATE_RESPONSE_CLEANUP_TIME_DELAY, + UPDATE_RESPONSE_CLEANUP_TIME_MILLS, + ARCHIVE_UPDATE_RESPONSES_FOLDER, +} from '../../constants/constants.js'; + +class UpdateResponseCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperationResponse( + nowTimestamp - UPDATE_RESPONSE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS.UPDATE, + ); + } + + getArchiveFolderName() { + return ARCHIVE_UPDATE_RESPONSES_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationResponse(ids, OPERATIONS.UPDATE); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'updateResponseCleanerCommand', + data: {}, + period: UPDATE_RESPONSE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default UpdateResponseCleanerCommand; diff --git a/src/commands/command-executor.js b/src/commands/command-executor.js index e44734f716..e62e042acb 100644 --- a/src/commands/command-executor.js +++ b/src/commands/command-executor.js @@ -14,14 +14,12 @@ import { */ class CommandExecutor { constructor(ctx) { - this.ctx = ctx; this.logger = ctx.logger; this.commandResolver = ctx.commandResolver; - this.config = ctx.config; this.started = false; this.repositoryModuleManager = ctx.repositoryModuleManager; - this.verboseLoggingEnabled = this.config.commandExecutorVerboseLoggingEnabled; + this.verboseLoggingEnabled = ctx.config.commandExecutorVerboseLoggingEnabled; this.queue = async.queue((command, callback = () => {}) => { this._execute(command) @@ -90,8 +88,8 @@ class CommandExecutor { }); try { const result = await handler.expired(command); - if (result && result.commands) { - result.commands.forEach((c) => this.add(c, c.delay, true)); + if (result?.commands) { + await Promise.all(result.commands.map((c) => this.add(c, c.delay, true))); } } catch (e) { this.logger.warn( @@ -135,9 +133,7 @@ class CommandExecutor { command.data = handler.pack(command.data); - const period = command.period - ? command.period - : DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS; + const period = command.period ?? DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS; await this.add(command, period, false); return Command.repeat(); } @@ -187,8 +183,18 @@ class CommandExecutor { try { const result = await this._handleError(command, handler, e); - if (result && result.commands) { - result.commands.forEach((c) => this.add(c, c.delay, true)); + if (result && result.repeat) { + await this._update(command, { + status: COMMAND_STATUS.REPEATING, + }); + + command.data = handler.pack(command.data); + + const period = command.period + ? command.period + : DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS; + await this.add(command, period, false); + return Command.repeat(); } } catch (error) { this.logger.warn( @@ -237,14 +243,13 @@ class CommandExecutor { * @param delay * @param insert */ - async add(addCommand, addDelay = 0, insert = true) { + async add(addCommand, addDelay, insert = true) { let command = addCommand; - let delay = addDelay; - const now = Date.now(); + let delay = addDelay ?? 0; - if (delay != null && delay > MAX_COMMAND_DELAY_IN_MILLS) { + if (delay > MAX_COMMAND_DELAY_IN_MILLS) { if (command.readyAt == null) { - command.readyAt = now; + command.readyAt = Date.now(); } command.readyAt += delay; delay = MAX_COMMAND_DELAY_IN_MILLS; @@ -280,8 +285,8 @@ class CommandExecutor { status: COMMAND_STATUS.PENDING, retries: command.retries - 1, }); - const period = command.period ? command.period : 0; - const delay = command.delay ? command.delay : 0; + const period = command.period ?? 0; + const delay = command.delay ?? 0; await this.add(command, period + delay, false); return Command.retry(); } @@ -302,8 +307,8 @@ class CommandExecutor { await this._update(command, { retries: command.retries - 1, }); - const period = command.period ? command.period : 0; - const delay = command.delay ? command.delay : 0; + const period = command.period ?? 0; + const delay = command.delay ?? 0; await this.add(command, period + delay, false); } else { try { @@ -353,7 +358,7 @@ class CommandExecutor { opts.transaction = transaction; } const model = await this.repositoryModuleManager.createCommand(command, opts); - command.id = model.dataValues.id; + command.id = model.id; return command; } @@ -394,49 +399,50 @@ class CommandExecutor { * @returns {Promise} */ async replay() { - // Wait for 1 minute for node to establish connections - // await new Promise((resolve) => setTimeout(resolve, 1 * 60 * 1000)); - this.logger.info('Replay pending/started commands from the database...'); - const pendingCommands = ( - await this.repositoryModuleManager.getCommandsWithStatus( - [COMMAND_STATUS.PENDING, COMMAND_STATUS.STARTED, COMMAND_STATUS.REPEATING], - ['cleanerCommand', 'autoupdaterCommand'], - ) - ).filter((command) => !PERMANENT_COMMANDS.includes(command.name)); - - // TODO consider JOIN instead - const commands = pendingCommands.filter(async (pc) => { - if (!pc.parentId) { - return true; + const pendingCommands = await this.repositoryModuleManager.getCommandsWithStatus( + [COMMAND_STATUS.PENDING, COMMAND_STATUS.STARTED, COMMAND_STATUS.REPEATING], + PERMANENT_COMMANDS, + ); + + const commands = []; + for (const command of pendingCommands) { + if (!command?.parentId) { + continue; } - const parent = await this.repositoryModuleManager.getCommandWithId(pc.parentId); - return !parent || parent.status === 'COMPLETED'; - }); + + // eslint-disable-next-line no-await-in-loop + const parent = await this.repositoryModuleManager.getCommandWithId(command.parentId); + if (parent && parent.status !== 'COMPLETED') { + continue; + } + commands.push(command); + } const adds = []; for (const commandModel of commands) { - const command = { - id: commandModel.id, - name: commandModel.name, - data: commandModel.data, - readyAt: commandModel.readyAt, - delay: commandModel.delay, - startedAt: commandModel.startedAt, - deadlineAt: commandModel.deadlineAt, - period: commandModel.period, - status: commandModel.status, - message: commandModel.message, - parentId: commandModel.parentId, - transactional: commandModel.transactional, - retries: commandModel.retries, - sequence: commandModel.sequence, - }; - const queued = this.queue.workersList().find((e) => e.data.id === command.id); + const queued = this.queue.workersList().find((e) => e.data.id === commandModel.id); if (!queued) { - adds.push(this.add(command, 0, false, true)); + const command = { + id: commandModel.id, + name: commandModel.name, + data: commandModel.data, + readyAt: commandModel.readyAt, + delay: commandModel.delay, + startedAt: commandModel.startedAt, + deadlineAt: commandModel.deadlineAt, + period: commandModel.period, + status: commandModel.status, + message: commandModel.message, + parentId: commandModel.parentId, + transactional: commandModel.transactional, + retries: commandModel.retries, + sequence: commandModel.sequence, + }; + adds.push(this.add(command, 0, false)); } } + await Promise.all(adds); } } diff --git a/src/commands/common/commands-cleaner-command.js b/src/commands/common/commands-cleaner-command.js deleted file mode 100644 index a1f214eecf..0000000000 --- a/src/commands/common/commands-cleaner-command.js +++ /dev/null @@ -1,57 +0,0 @@ -import Command from '../command.js'; -// eslint-disable-next-line no-unused-vars -import { COMMAND_STATUS, FINALIZED_COMMAND_CLEANUP_TIME_MILLS } from '../../constants/constants.js'; - -/** - * Increases approval for Bidding contract on blockchain - */ -class CommandsCleanerCommand extends Command { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.repositoryModuleManager = ctx.repositoryModuleManager; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute() { - // TODO: Uncomment after discussion - // await this.repositoryModuleManager.removeFinalizedCommands([ - // COMMAND_STATUS.COMPLETED, - // COMMAND_STATUS.FAILED, - // COMMAND_STATUS.EXPIRED, - // COMMAND_STATUS.UNKNOWN, - // ]); - return Command.repeat(); - } - - /** - * Recover system from failure - * @param command - * @param error - */ - async recover(command, error) { - this.logger.warn(`Failed to clean finalized commands: error: ${error.message}`); - return Command.repeat(); - } - - /** - * Builds default command - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'commandsCleanerCommand', - data: {}, - period: FINALIZED_COMMAND_CLEANUP_TIME_MILLS, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default CommandsCleanerCommand; diff --git a/src/commands/local-store/local-store-command.js b/src/commands/local-store/local-store-command.js index 7a8f5c8db4..8719f8d322 100644 --- a/src/commands/local-store/local-store-command.js +++ b/src/commands/local-store/local-store-command.js @@ -80,6 +80,7 @@ class LocalStoreCommand extends Command { blockchain, contract, tokenId, + cachedData.public.assertionId, { ...cachedData, keyword, diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 1ab33426ff..3d674f498b 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -1,10 +1,10 @@ /* eslint-disable no-await-in-loop */ -import { v4 as uuidv4 } from 'uuid'; import Command from '../../command.js'; import { COMMAND_QUEUE_PARALLELISM, COMMAND_RETRIES, TRANSACTION_CONFIRMATIONS, + OPERATION_ID_STATUS, } from '../../../constants/constants.js'; class EpochCheckCommand extends Command { @@ -19,6 +19,11 @@ class EpochCheckCommand extends Command { } async execute(command) { + const operationId = this.operationIdService.generateId(); + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.EPOCH_CHECK_START, + operationId, + ); await Promise.all( this.blockchainModuleManager.getImplementationNames().map(async (blockchain) => { const commitWindowDurationPerc = @@ -40,26 +45,47 @@ class EpochCheckCommand extends Command { const transactionQueueLength = this.blockchainModuleManager.getTransactionQueueLength(blockchain); if (transactionQueueLength >= totalTransactions) return; + totalTransactions -= transactionQueueLength; + const [r0, r2] = await Promise.all([ + this.blockchainModuleManager.getR0(blockchain), + this.blockchainModuleManager.getR2(blockchain), + ]); + await Promise.all([ this.scheduleSubmitCommitCommands( blockchain, Math.floor(totalTransactions / 2), commitWindowDurationPerc, + r0, + r2, ), this.scheduleCalculateProofsCommands( blockchain, Math.ceil(totalTransactions / 2), proofWindowDurationPerc, + r0, ), ]); }), ); + + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.EPOCH_CHECK_END, + operationId, + ); + return Command.repeat(); } - async scheduleSubmitCommitCommands(blockchain, maxTransactions, commitWindowDurationPerc) { + async scheduleSubmitCommitCommands( + blockchain, + maxTransactions, + commitWindowDurationPerc, + r0, + r2, + ) { const timestamp = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); const eligibleAgreementForSubmitCommit = await this.repositoryModuleManager.getEligibleAgreementsForSubmitCommit( @@ -68,9 +94,6 @@ class EpochCheckCommand extends Command { commitWindowDurationPerc, ); - const r0 = await this.blockchainModuleManager.getR0(blockchain); - const r2 = await this.blockchainModuleManager.getR2(blockchain); - const scheduleSubmitCommitCommands = []; const updateServiceAgreementsLastCommitEpoch = []; for (const serviceAgreement of eligibleAgreementForSubmitCommit) { @@ -120,7 +143,12 @@ class EpochCheckCommand extends Command { ]); } - async scheduleCalculateProofsCommands(blockchain, maxTransactions, proofWindowDurationPerc) { + async scheduleCalculateProofsCommands( + blockchain, + maxTransactions, + proofWindowDurationPerc, + r0, + ) { const timestamp = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); const eligibleAgreementsForSubmitProofs = await this.repositoryModuleManager.getEligibleAgreementsForSubmitProof( @@ -138,6 +166,7 @@ class EpochCheckCommand extends Command { serviceAgreement.agreementId, serviceAgreement.currentEpoch, serviceAgreement.stateIndex, + r0, ); if (eligibleForReward) { this.logger.trace( @@ -196,8 +225,7 @@ class EpochCheckCommand extends Command { return scores.findIndex((node) => node.peerId === peerId); } - async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex) { - const r0 = await this.blockchainModuleManager.getR0(blockchain); + async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex, r0) { const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, @@ -217,7 +245,7 @@ class EpochCheckCommand extends Command { async scheduleSubmitCommitCommand(agreement) { const commandData = { - operationId: uuidv4(), + operationId: this.operationIdService.generateId(), blockchain: agreement.blockchainId, contract: agreement.assetStorageContractAddress, tokenId: agreement.tokenId, @@ -239,7 +267,7 @@ class EpochCheckCommand extends Command { async scheduleSubmitProofsCommand(agreement) { const commandData = { - operationId: uuidv4(), + operationId: this.operationIdService.generateId(), blockchain: agreement.blockchainId, contract: agreement.assetStorageContractAddress, tokenId: agreement.tokenId, diff --git a/src/commands/protocols/common/handle-protocol-message-command.js b/src/commands/protocols/common/handle-protocol-message-command.js index 36e560dcc8..43b74e4e8a 100644 --- a/src/commands/protocols/common/handle-protocol-message-command.js +++ b/src/commands/protocols/common/handle-protocol-message-command.js @@ -38,6 +38,8 @@ class HandleProtocolMessageCommand extends Command { await this.handleError(error.message, command); } + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, remotePeerId); + return Command.empty(); } @@ -223,6 +225,7 @@ class HandleProtocolMessageCommand extends Command { keywordUuid, { errorMessage }, ); + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, remotePeerId); } } diff --git a/src/commands/protocols/common/protocol-message-command.js b/src/commands/protocols/common/protocol-message-command.js index 5907dd0554..7e3c2a0424 100644 --- a/src/commands/protocols/common/protocol-message-command.js +++ b/src/commands/protocols/common/protocol-message-command.js @@ -1,9 +1,6 @@ +import { v5 as uuidv5 } from 'uuid'; import Command from '../../command.js'; -import { - NETWORK_MESSAGE_TYPES, - OPERATION_REQUEST_STATUS, - OPERATION_STATUS, -} from '../../../constants/constants.js'; +import { NETWORK_MESSAGE_TYPES, OPERATION_REQUEST_STATUS } from '../../../constants/constants.js'; class ProtocolMessageCommand extends Command { constructor(ctx) { @@ -20,19 +17,8 @@ class ProtocolMessageCommand extends Command { return this.sendProtocolMessage(command, message, messageType); } - async shouldSendMessage(command) { - const { operationId } = command.data; - - const { status } = await this.operationService.getOperationStatus(operationId); - - if (status === OPERATION_STATUS.IN_PROGRESS) { - return true; - } - this.logger.trace( - `${command.name} skipped for operationId: ${operationId} with status ${status}`, - ); - - return false; + async shouldSendMessage() { + return true; } async prepareMessage() { @@ -42,16 +28,20 @@ class ProtocolMessageCommand extends Command { async sendProtocolMessage(command, message, messageType) { const { node, operationId, keyword } = command.data; + const keywordUuid = uuidv5(keyword, uuidv5.URL); + const response = await this.networkModuleManager.sendMessage( node.protocol, node.id, messageType, operationId, - keyword, + keywordUuid, message, this.messageTimeout(), ); + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, node.id); + switch (response.header.messageType) { case NETWORK_MESSAGE_TYPES.RESPONSES.BUSY: return this.handleBusy(command, response.data); @@ -89,6 +79,10 @@ class ProtocolMessageCommand extends Command { } async recover(command, err) { + const { node, operationId, keyword } = command.data; + const keywordUuid = uuidv5(keyword, uuidv5.URL); + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, node.id); + await this.markResponseAsFailed(command, err.message); return Command.empty(); } diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js index 48d3b3decd..e8bcd7aee4 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js @@ -19,7 +19,7 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { } async prepareMessage(commandData) { - const { assertionId, operationId, state, blockchain, contract, tokenId } = commandData; + const { operationId, blockchain, contract, tokenId, assertionId, state } = commandData; await this.operationIdService.updateOperationIdStatus( operationId, @@ -32,22 +32,30 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { let assertionExists; if ( - state === GET_STATES.LATEST && + state !== GET_STATES.FINALIZED && blockchain != null && contract != null && tokenId != null ) { - assertionExists = await this.pendingStorageService.assertionExists( + assertionExists = await this.pendingStorageService.assetHasPendingState( PENDING_STORAGE_REPOSITORIES.PUBLIC, blockchain, contract, tokenId, - operationId, ); } - if (!assertionExists) { + + for (const repository of [ + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, + ]) { + if (assertionExists) { + break; + } + + // eslint-disable-next-line no-await-in-loop assertionExists = await this.tripleStoreService.assertionExists( - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + repository, assertionId, ); } diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js index 5f63b8011e..55adc3989e 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js @@ -20,23 +20,23 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { } async prepareMessage(commandData) { - const { assertionId, operationId, state } = commandData; + const { operationId, blockchain, contract, tokenId, assertionId, state } = commandData; await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.GET.GET_REMOTE_START, ); if ( - state === GET_STATES.LATEST && - commandData.blockchain != null && - commandData.contract != null && - commandData.tokenId != null + state !== GET_STATES.FINALIZED && + blockchain != null && + contract != null && + tokenId != null ) { const cachedAssertion = await this.pendingStorageService.getCachedAssertion( PENDING_STORAGE_REPOSITORIES.PUBLIC, - commandData.blockchain, - commandData.contract, - commandData.tokenId, + blockchain, + contract, + tokenId, operationId, ); if (cachedAssertion?.public?.assertion?.length) { @@ -47,10 +47,18 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { } } - const nquads = await this.tripleStoreService.getAssertion( + let nquads; + for (const repository of [ TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - assertionId, - ); + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, + ]) { + // eslint-disable-next-line no-await-in-loop + nquads = await this.tripleStoreService.getAssertion(repository, assertionId); + + if (nquads.length) { + break; + } + } await this.operationIdService.updateOperationIdStatus( operationId, diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js new file mode 100644 index 0000000000..b4af9d93d3 --- /dev/null +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -0,0 +1,104 @@ +import Command from '../../../command.js'; +import { ERROR_TYPE, GET_STATES, ZERO_BYTES32 } from '../../../../constants/constants.js'; + +class GetAssertionIdCommand extends Command { + constructor(ctx) { + super(ctx); + this.operationService = ctx.getService; + this.blockchainModuleManager = ctx.blockchainModuleManager; + this.ualService = ctx.ualService; + + this.errorType = ERROR_TYPE.GET.GET_ASSERTION_ID_ERROR; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute(command) { + const { operationId, blockchain, contract, tokenId, state } = command.data; + + let assertionId; + if (!Object.values(GET_STATES).includes(state)) { + if (state === ZERO_BYTES32) { + await this.handleError( + operationId, + `The provided state: ${state}. State hash cannot be 0x0.`, + this.errorType, + ); + + return Command.empty(); + } + + const pendingState = await this.blockchainModuleManager.getUnfinalizedAssertionId( + blockchain, + tokenId, + ); + + if ( + state !== pendingState && + !( + await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ) + ).includes(state) + ) { + await this.handleError( + operationId, + `The provided state: ${state} does not exist on the ${blockchain} blockchain, ``within contract: ${contract}, for the Knowledge Asset with tokenId: ${tokenId}.`, + this.errorType, + ); + + return Command.empty(); + } + + assertionId = state; + } else { + this.logger.debug( + `Searching for latest assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, + ); + + if (state === GET_STATES.LATEST) { + assertionId = await this.blockchainModuleManager.getUnfinalizedAssertionId( + blockchain, + tokenId, + ); + } + if (assertionId == null || assertionId === ZERO_BYTES32) { + assertionId = await this.blockchainModuleManager.getLatestAssertionId( + blockchain, + contract, + tokenId, + ); + } + } + + return this.continueSequence( + { ...command.data, state: assertionId, assertionId }, + command.sequence, + ); + } + + async handleError(operationId, errorMessage, errorType) { + await this.operationService.markOperationAsFailed(operationId, errorMessage, errorType); + } + + /** + * Builds default getStateIdConditionalCommand + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'getAssertionIdCommand', + delay: 0, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default GetAssertionIdCommand; diff --git a/src/commands/protocols/get/sender/get-latest-assertion-id-command.js b/src/commands/protocols/get/sender/get-latest-assertion-id-command.js deleted file mode 100644 index 362e66413c..0000000000 --- a/src/commands/protocols/get/sender/get-latest-assertion-id-command.js +++ /dev/null @@ -1,94 +0,0 @@ -import Command from '../../../command.js'; -import { ERROR_TYPE, GET_STATES } from '../../../../constants/constants.js'; - -class GetLatestAssertionIdCommand extends Command { - constructor(ctx) { - super(ctx); - this.operationService = ctx.getService; - this.blockchainModuleManager = ctx.blockchainModuleManager; - this.ualService = ctx.ualService; - - this.errorType = ERROR_TYPE.GET.GET_ASSERTION_ID_ERROR; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute(command) { - const { id, operationId, state } = command.data; - - const commandData = {}; - if (!this.ualService.isUAL(id)) { - this.handleError(operationId, `Requested id is not a UAL`, this.errorType, true); - - return Command.empty(); - } - - const { blockchain, contract, tokenId } = this.ualService.resolveUAL(id); - commandData.blockchain = blockchain; - commandData.tokenId = tokenId; - commandData.contract = contract; - - let unfinalizedAssertionId; - if (state === GET_STATES.LATEST) { - this.logger.debug( - `Searching for latest assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, - ); - unfinalizedAssertionId = await this.blockchainModuleManager.getUnfinalizedAssertionId( - blockchain, - tokenId, - ); - commandData.assertionId = unfinalizedAssertionId; - } - - if ( - typeof unfinalizedAssertionId === 'undefined' || - !unfinalizedAssertionId || - parseInt(unfinalizedAssertionId, 16) === 0 - ) { - this.logger.debug( - `Searching for latest finalized assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, - ); - const blockchainAssertionId = await this.blockchainModuleManager.getLatestAssertionId( - blockchain, - contract, - tokenId, - ); - if (!blockchainAssertionId) { - this.handleError( - operationId, - `Unable to find latest finalized assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, - this.errorType, - true, - ); - - return Command.empty(); - } - commandData.assertionId = blockchainAssertionId; - } - - return this.continueSequence({ ...command.data, ...commandData }, command.sequence); - } - - async handleError(operationId, errorMessage, errorType) { - await this.operationService.markOperationAsFailed(operationId, errorMessage, errorType); - } - - /** - * Builds default getLatestAssertionIdCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'getLatestAssertionIdCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default GetLatestAssertionIdCommand; diff --git a/src/commands/protocols/get/sender/local-get-command.js b/src/commands/protocols/get/sender/local-get-command.js index ec3b22e76d..f9d09cf0f8 100644 --- a/src/commands/protocols/get/sender/local-get-command.js +++ b/src/commands/protocols/get/sender/local-get-command.js @@ -2,7 +2,6 @@ import Command from '../../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, - GET_STATES, TRIPLE_STORE_REPOSITORIES, PENDING_STORAGE_REPOSITORIES, } from '../../../../constants/constants.js'; @@ -10,6 +9,7 @@ import { class LocalGetCommand extends Command { constructor(ctx) { super(ctx); + this.blockchainModuleManager = ctx.blockchainModuleManager; this.config = ctx.config; this.operationService = ctx.getService; this.operationIdService = ctx.operationIdService; @@ -24,31 +24,36 @@ class LocalGetCommand extends Command { * @param command */ async execute(command) { - const { operationId, assertionId, state } = command.data; + const { operationId, blockchain, contract, tokenId, state } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.GET.GET_LOCAL_START, ); const response = {}; - if ( - state === GET_STATES.LATEST && - command.data.blockchain != null && - command.data.contract != null && - command.data.tokenId != null - ) { - for (const repository of [ - PENDING_STORAGE_REPOSITORIES.PRIVATE, - PENDING_STORAGE_REPOSITORIES.PUBLIC, - ]) { + for (const repository of [ + PENDING_STORAGE_REPOSITORIES.PRIVATE, + PENDING_STORAGE_REPOSITORIES.PUBLIC, + ]) { + // eslint-disable-next-line no-await-in-loop + const stateIsPending = await this.pendingStorageService.assetHasPendingState( + repository, + blockchain, + contract, + tokenId, + state, + ); + + if (stateIsPending) { // eslint-disable-next-line no-await-in-loop const cachedAssertion = await this.pendingStorageService.getCachedAssertion( repository, - command.data.blockchain, - command.data.contract, - command.data.tokenId, + blockchain, + contract, + tokenId, operationId, ); + if (cachedAssertion?.public?.assertion?.length) { response.assertion = cachedAssertion.public.assertion; if (cachedAssertion?.private?.assertion?.length) { @@ -63,12 +68,11 @@ class LocalGetCommand extends Command { for (const repository of [ TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT, TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + TRIPLE_STORE_REPOSITORIES.PRIVATE_HISTORY, + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, ]) { // eslint-disable-next-line no-await-in-loop - response.assertion = await this.tripleStoreService.getAssertion( - repository, - assertionId, - ); + response.assertion = await this.tripleStoreService.getAssertion(repository, state); if (response?.assertion?.length) break; } } diff --git a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js index c32b436f5b..ce55bf1ace 100644 --- a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js +++ b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js @@ -10,8 +10,16 @@ class GetInitCommand extends ProtocolInitCommand { } async prepareMessage(command) { - const commandData = await super.prepareMessage(command); - return { ...commandData, state: command.data.state }; + const { blockchain, contract, tokenId, keyword, assertionId, state } = command.data; + + return { + blockchain, + contract, + tokenId, + keyword, + assertionId, + state, + }; } messageTimeout() { diff --git a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js index 506644c953..7c6d6e1cfc 100644 --- a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js +++ b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js @@ -3,6 +3,7 @@ import { NETWORK_MESSAGE_TIMEOUT_MILLS, ERROR_TYPE, OPERATION_REQUEST_STATUS, + OPERATION_STATUS, } from '../../../../../constants/constants.js'; class GetRequestCommand extends ProtocolRequestCommand { @@ -14,16 +15,31 @@ class GetRequestCommand extends ProtocolRequestCommand { this.errorType = ERROR_TYPE.GET.GET_REQUEST_ERROR; } + async shouldSendMessage(command) { + const { operationId } = command.data; + + const { status } = await this.operationService.getOperationStatus(operationId); + + if (status === OPERATION_STATUS.IN_PROGRESS) { + return true; + } + this.logger.trace( + `${command.name} skipped for operationId: ${operationId} with status ${status}`, + ); + + return false; + } + async prepareMessage(command) { - const { assertionId, blockchain, contract, tokenId, hashFunctionId, state } = command.data; + const { blockchain, contract, tokenId, assertionId, state, hashFunctionId } = command.data; return { - assertionId, blockchain, contract, tokenId, - hashFunctionId, + assertionId, state, + hashFunctionId, }; } diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index c26f461a4b..83eff51b71 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -42,15 +42,16 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { OPERATION_ID_STATUS.UPDATE.VALIDATING_UPDATE_ASSERTION_REMOTE_START, ); - const { assertion } = await this.operationIdService.getCachedOperationIdData(operationId); + const cachedData = await this.operationIdService.getCachedOperationIdData(operationId); await this.pendingStorageService.cacheAssertion( PENDING_STORAGE_REPOSITORIES.PUBLIC, blockchain, contract, tokenId, + cachedData.assertionId, { public: { - assertion, + assertion: cachedData.assertion, }, agreementId, agreementData, diff --git a/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js b/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js index eb89dd39dc..cda7aec635 100644 --- a/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js +++ b/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js @@ -10,10 +10,9 @@ class UpdateRequestCommand extends ProtocolRequestCommand { } async prepareMessage(command) { - const data = await this.operationIdService.getCachedOperationIdData( - command.data.operationId, - ); - const { assertion } = data.public; + const { + public: { assertion }, + } = await this.operationIdService.getCachedOperationIdData(command.data.operationId); return { assertion, diff --git a/src/constants/constants.js b/src/constants/constants.js index 611ded8315..9c807292ea 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -8,6 +8,8 @@ export const STAKE_UINT256_MULTIPLIER_BN = UINT256_MAX_BN.div(500000000); export const UINT256_UINT32_DIVISOR_BN = UINT256_MAX_BN.div(UINT32_MAX_BN); +export const ZERO_BYTES32 = `0x${'0'.repeat(64)}`; + export const SCHEMA_CONTEXT = 'http://schema.org/'; export const PRIVATE_ASSERTION_PREDICATE = @@ -27,7 +29,7 @@ export const TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; export const DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS = 15 * 24 * 60 * 60 * 1000; // 15 days -export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 500; +export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 50; export const TRANSACTION_QUEUE_CONCURRENCY = 1; @@ -35,7 +37,7 @@ export const TRIPLE_STORE_CONNECT_RETRY_FREQUENCY = 10; export const MAX_FILE_SIZE = 2621440; -export const GET_STATES = { LATEST: 'LATEST', LATEST_FINALIZED: 'LATEST_FINALIZED' }; +export const GET_STATES = { LATEST: 'LATEST', FINALIZED: 'LATEST_FINALIZED' }; export const BYTES_IN_KILOBYTE = 1024; export const BYTES_IN_MEGABYTE = BYTES_IN_KILOBYTE * BYTES_IN_KILOBYTE; @@ -95,12 +97,12 @@ export const MIN_NODE_VERSION = 16; export const NETWORK_API_RATE_LIMIT = { TIME_WINDOW_MILLS: 1 * 60 * 1000, - MAX_NUMBER: 20, + MAX_NUMBER: 100, }; export const NETWORK_API_SPAM_DETECTION = { TIME_WINDOW_MILLS: 1 * 60 * 1000, - MAX_NUMBER: 40, + MAX_NUMBER: 150, }; export const NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES = 60; @@ -130,6 +132,13 @@ export const PERMANENT_COMMANDS = [ 'commandsCleanerCommand', 'dialPeersCommand', 'epochCheckCommand', + 'blockchainEventCleanerCommand', + 'getCleanerCommand', + 'getResponseCleanerCommand', + 'publishCleanerCommand', + 'publishResponseCleanerCommand', + 'updateCleanerCommand', + 'updateResponseCleanerCommand', ]; export const MAX_COMMAND_DELAY_IN_MILLS = 14400 * 60 * 1000; // 10 days @@ -336,7 +345,35 @@ export const OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; * @constant {number} FINALIZED_COMMAND_CLEANUP_TIME_MILLS - Command cleanup interval time * finalized commands command cleanup interval time 24h */ -export const FINALIZED_COMMAND_CLEANUP_TIME_MILLS = 30 * 24 * 60 * 60 * 1000; +export const FINALIZED_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const GET_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const GET_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const GET_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const GET_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const PUBLISH_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const PUBLISH_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const PUBLISH_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const PUBLISH_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const UPDATE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const UPDATE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const UPDATE_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const UPDATE_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; /** * @constant {number} COMMAND_STATUS - * Status for commands @@ -351,6 +388,26 @@ export const COMMAND_STATUS = { REPEATING: 'REPEATING', }; +export const OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER = 100; + +export const REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER = 1000; + +export const ARCHIVE_COMMANDS_FOLDER = 'commands'; + +export const ARCHIVE_BLOCKCHAIN_EVENTS_FOLDER = 'blockchain_events'; + +export const ARCHIVE_GET_FOLDER = 'get'; + +export const ARCHIVE_GET_RESPONSES_FOLDER = 'get_responses'; + +export const ARCHIVE_PUBLISH_FOLDER = 'publish'; + +export const ARCHIVE_PUBLISH_RESPONSES_FOLDER = 'publish_responses'; + +export const ARCHIVE_UPDATE_FOLDER = 'update'; + +export const ARCHIVE_UPDATE_RESPONSES_FOLDER = 'update_responses'; + /** * How many commands will run in parallel * @type {number} @@ -415,30 +472,12 @@ export const CONTRACTS = { }; export const CONTRACT_EVENTS = { - HUB: { - NEW_CONTRACT: 'NewContract', - CONTRACT_CHANGED: 'ContractChanged', - NEW_ASSET_STORAGE: 'NewAssetStorage', - ASSET_STORAGE_CHANGED: 'AssetStorageChanged', - }, - SHARDING_TABLE: { - NODE_ADDED: 'NodeAdded', - NODE_REMOVED: 'NodeRemoved', - }, - STAKING: { - STAKE_INCREASED: 'StakeIncreased', - STAKE_WITHDRAWAL_STARTED: 'StakeWithdrawalStarted', - }, - PROFILE: { - ASK_UPDATED: 'AskUpdated', - }, - COMMIT_MANAGER_V1: { - STATE_FINALIZED: 'StateFinalized', - }, - SERVICE_AGREEMENT_V1: { - SERVICE_AGREEMENT_V1_EXTENDED: 'ServiceAgreementV1Extended', - SERVICE_AGREEMENT_V1_TERMINATED: 'ServiceAgreementV1Terminated', - }, + HUB: ['NewContract', 'ContractChanged', 'NewAssetStorage', 'AssetStorageChanged'], + SHARDING_TABLE: ['NodeAdded', 'NodeRemoved'], + STAKING: ['StakeIncreased', 'StakeWithdrawalStarted'], + PROFILE: ['AskUpdated'], + COMMIT_MANAGER_V1: ['StateFinalized'], + SERVICE_AGREEMENT_V1: ['ServiceAgreementV1Extended', 'ServiceAgreementV1Terminated'], }; export const NODE_ENVIRONMENTS = { diff --git a/src/controllers/http-api/get-http-api-controller.js b/src/controllers/http-api/get-http-api-controller.js index 0d5fd7ebae..8855bb0a99 100644 --- a/src/controllers/http-api/get-http-api-controller.js +++ b/src/controllers/http-api/get-http-api-controller.js @@ -14,6 +14,8 @@ class GetController extends BaseController { this.operationIdService = ctx.operationIdService; this.operationService = ctx.getService; this.repositoryModuleManager = ctx.repositoryModuleManager; + this.ualService = ctx.ualService; + this.validationService = ctx.validationService; } async handleGetRequest(req, res) { @@ -38,13 +40,29 @@ class GetController extends BaseController { try { const { id } = req.body; + + if (!this.ualService.isUAL(id)) { + throw Error('Requested id is not a UAL.'); + } + + const { blockchain, contract, tokenId } = this.ualService.resolveUAL(id); + + const isValidUal = await this.validationService.validateUal( + blockchain, + contract, + tokenId, + ); + if (!isValidUal) { + throw Error(`${id} UAL isn't valid.`); + } + const state = req.body.state ?? DEFAULT_GET_STATE; const hashFunctionId = req.body.hashFunctionId ?? CONTENT_ASSET_HASH_FUNCTION_ID; this.logger.info(`Get for ${id} with operation id ${operationId} initiated.`); const commandSequence = [ - 'getLatestAssertionIdCommand', + 'getAssertionIdCommand', 'localGetCommand', 'networkGetCommand', ]; @@ -54,8 +72,10 @@ class GetController extends BaseController { sequence: commandSequence.slice(1), delay: 0, data: { + blockchain, + contract, + tokenId, operationId, - id, state, hashFunctionId, }, diff --git a/src/controllers/http-api/request-schema/get-schema.js b/src/controllers/http-api/request-schema/get-schema.js index 6874ae720c..6d1d6b05a0 100644 --- a/src/controllers/http-api/request-schema/get-schema.js +++ b/src/controllers/http-api/request-schema/get-schema.js @@ -7,8 +7,14 @@ export default () => ({ id: { type: 'string', }, - type: { - enum: [GET_STATES.LATEST, GET_STATES.LATEST_FINALIZED], + state: { + oneOf: [ + { enum: [GET_STATES.LATEST, GET_STATES.FINALIZED] }, + { + type: 'string', + pattern: '^0x[A-Fa-f0-9]{64}$', + }, + ], }, hashFunctionId: { type: 'number', diff --git a/src/controllers/rpc/publish-rpc-controller.js b/src/controllers/rpc/publish-rpc-controller.js index 2d02db662b..84010abb3c 100644 --- a/src/controllers/rpc/publish-rpc-controller.js +++ b/src/controllers/rpc/publish-rpc-controller.js @@ -30,6 +30,7 @@ class PublishController extends BaseController { // eslint-disable-next-line no-case-declarations dataSource = await this.operationIdService.getCachedOperationIdData(operationId); await this.operationIdService.cacheOperationIdData(operationId, { + assertionId: dataSource.assertionId, assertion: message.data.assertion, }); command.name = handleRequestCommand; diff --git a/src/controllers/rpc/update-rpc-controller.js b/src/controllers/rpc/update-rpc-controller.js index 7cc6a795cd..f9fd8764d5 100644 --- a/src/controllers/rpc/update-rpc-controller.js +++ b/src/controllers/rpc/update-rpc-controller.js @@ -1,5 +1,8 @@ import BaseController from './base-rpc-controller.js'; -import { CONTENT_ASSET_HASH_FUNCTION_ID, NETWORK_MESSAGE_TYPES } from '../../constants/constants.js'; +import { + CONTENT_ASSET_HASH_FUNCTION_ID, + NETWORK_MESSAGE_TYPES, +} from '../../constants/constants.js'; class UpdateController extends BaseController { constructor(ctx) { @@ -27,6 +30,7 @@ class UpdateController extends BaseController { // eslint-disable-next-line no-case-declarations dataSource = await this.operationIdService.getCachedOperationIdData(operationId); await this.operationIdService.cacheOperationIdData(operationId, { + assertionId: dataSource.assertionId, assertion: message.data.assertion, }); command.name = handleRequestCommand; diff --git a/src/migration/base-migration.js b/src/migration/base-migration.js index 3755f03a55..0b17fa6577 100644 --- a/src/migration/base-migration.js +++ b/src/migration/base-migration.js @@ -27,7 +27,7 @@ class BaseMigration { this.fileService.getMigrationFolderPath(), this.migrationName, ); - if (await this.fileService.fileExists(migrationFilePath)) { + if (await this.fileService.pathExists(migrationFilePath)) { return true; } return false; diff --git a/src/migration/blockchain-identity-migration.js b/src/migration/blockchain-identity-migration.js index 6086ab61ad..fb697a1ee0 100644 --- a/src/migration/blockchain-identity-migration.js +++ b/src/migration/blockchain-identity-migration.js @@ -15,7 +15,7 @@ class BlockchainIdentityMigration extends BaseMigration { this.config.configFilename, ); - const config = await this.fileService.loadJsonFromFile(configurationFilePath); + const config = await this.fileService.readFile(configurationFilePath, true); for (const blockchainImpl in config.modules.blockchain.implementation) { delete config.modules.blockchain.implementation[blockchainImpl].config.identity; } diff --git a/src/migration/pending-storage-migration.js b/src/migration/pending-storage-migration.js new file mode 100644 index 0000000000..040cf491a5 --- /dev/null +++ b/src/migration/pending-storage-migration.js @@ -0,0 +1,38 @@ +import path from 'path'; +import { calculateRoot } from 'assertion-tools'; +import { PENDING_STORAGE_REPOSITORIES } from '../constants/constants.js'; +import BaseMigration from './base-migration.js'; + +class PendingStorageMigration extends BaseMigration { + async executeMigration() { + const promises = Object.values(PENDING_STORAGE_REPOSITORIES).map(async (repository) => { + let fileNames; + const repositoryPath = this.fileService.getPendingStorageCachePath(repository); + try { + fileNames = await this.fileService.readDirectory(repositoryPath); + } catch (error) { + return false; + } + + await Promise.all( + fileNames.map(async (fileName) => { + const newDirectoryPath = path.join(repositoryPath, fileName); + const cachedData = await this.fileService.readFile(newDirectoryPath, true); + await this.fileService.removeFile(newDirectoryPath); + if (cachedData?.public?.assertion) { + const newDocumentName = calculateRoot(cachedData.public.assertion); + await this.fileService.writeContentsToFile( + newDirectoryPath, + newDocumentName, + JSON.stringify(cachedData), + ); + } + }), + ); + }); + + await Promise.all(promises); + } +} + +export default PendingStorageMigration; diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js index bcfbbeb9b5..8c29a8771c 100644 --- a/src/migration/service-agreements-metadata-migration.js +++ b/src/migration/service-agreements-metadata-migration.js @@ -32,9 +32,9 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { const migrationInfoFileName = `${this.migrationName}_info`; const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); let migrationInfo; - if (await this.fileService.fileExists(migrationInfoPath)) { + if (await this.fileService.pathExists(migrationInfoPath)) { migrationInfo = await this.fileService - ._readFile(migrationInfoPath, true) + .readFile(migrationInfoPath, true) .catch(() => {}); } if (!migrationInfo?.lastProcessedTokenId) { diff --git a/src/migration/triple-store-metadata-migration.js b/src/migration/triple-store-metadata-migration.js index ffdcc87cc6..dc087689d8 100644 --- a/src/migration/triple-store-metadata-migration.js +++ b/src/migration/triple-store-metadata-migration.js @@ -36,9 +36,9 @@ class TripleStoreMetadataMigration extends BaseMigration { const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); let migrationInfo; - if (await this.fileService.fileExists(migrationInfoPath)) { + if (await this.fileService.pathExists(migrationInfoPath)) { try { - migrationInfo = await this.fileService._readFile(migrationInfoPath, true); + migrationInfo = await this.fileService.readFile(migrationInfoPath, true); } catch (error) { migrationInfo = { status: 'IN_PROGRESS', @@ -82,9 +82,9 @@ class TripleStoreMetadataMigration extends BaseMigration { const migrationInfoFileName = `${this.migrationName}_${currentRepository}`; const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); let migrationInfo; - if (await this.fileService.fileExists(migrationInfoPath)) { + if (await this.fileService.pathExists(migrationInfoPath)) { try { - migrationInfo = await this.fileService._readFile(migrationInfoPath, true); + migrationInfo = await this.fileService.readFile(migrationInfoPath, true); } catch (error) { migrationInfo = { status: 'IN_PROGRESS', diff --git a/src/migration/triple-store-user-configuration-migration.js b/src/migration/triple-store-user-configuration-migration.js index cb0c2c258d..ba6f7b8646 100644 --- a/src/migration/triple-store-user-configuration-migration.js +++ b/src/migration/triple-store-user-configuration-migration.js @@ -15,9 +15,7 @@ class TripleStoreUserConfigurationMigration extends BaseMigration { this.config.configFilename, ); - const userConfiguration = await this.fileService.loadJsonFromFile( - configurationFilePath, - ); + const userConfiguration = await this.fileService.readFile(configurationFilePath, true); if (userConfiguration.modules.tripleStore.implementation) { for (const implementationName in userConfiguration.modules.tripleStore .implementation) { @@ -91,8 +89,9 @@ class TripleStoreUserConfigurationMigration extends BaseMigration { 'local-network-setup', '.origintrail_noderc_template.json', ); - const configurationTemplate = await this.fileService.loadJsonFromFile( + const configurationTemplate = await this.fileService.readFile( configurationTemplatePath, + true, ); if ( diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 2079c13271..55ea81e810 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -115,6 +115,13 @@ class BlockchainModuleManager extends BaseModuleManager { ]); } + async getKnowledgeAssetOwner(blockchain, assetContractAddress, tokenId) { + return this.callImplementationFunction(blockchain, 'getKnowledgeAssetOwner', [ + assetContractAddress, + tokenId, + ]); + } + async getUnfinalizedAssertionId(blockchain, tokenId) { return this.callImplementationFunction(blockchain, 'getUnfinalizedState', [tokenId]); } @@ -141,6 +148,7 @@ class BlockchainModuleManager extends BaseModuleManager { async getAllPastEvents( blockchain, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -148,6 +156,7 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'getAllPastEvents', [ blockchain, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 54f122b55a..6a30def3b1 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -17,7 +17,7 @@ import { const require = createRequire(import.meta.url); const ABIs = { - AbstractAsset: require('dkg-evm-module/abi/AbstractAsset.json'), + ContentAssetStorage: require('dkg-evm-module/abi/ContentAssetStorage.json'), AssertionStorage: require('dkg-evm-module/abi/AssertionStorage.json'), Staking: require('dkg-evm-module/abi/Staking.json'), StakingStorage: require('dkg-evm-module/abi/StakingStorage.json'), @@ -171,7 +171,7 @@ class Web3Service { initializeAssetStorageContract(assetStorageAddress) { this.assetStorageContracts[assetStorageAddress.toLowerCase()] = new ethers.Contract( assetStorageAddress, - ABIs.AbstractAsset, + ABIs.ContentAssetStorage, this.wallet, ); } @@ -396,6 +396,7 @@ class Web3Service { async getAllPastEvents( blockchainId, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -412,14 +413,21 @@ class Web3Service { fromBlock = lastCheckedBlock + 1; } - let events = []; + const topics = []; + for (const filterName in contract.filters) { + if (!eventsToFilter.includes(filterName)) continue; + const filter = contract.filters[filterName]().topics[0]; + topics.push(filter); + } + + const events = []; while (fromBlock <= currentBlock) { const toBlock = Math.min( fromBlock + MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH - 1, currentBlock, ); - const newEvents = await contract.queryFilter('*', fromBlock, toBlock); - events = events.concat(newEvents); + const newEvents = await this.processBlockRange(fromBlock, toBlock, contract, topics); + newEvents.forEach((e) => events.push(...e)); fromBlock = toBlock + 1; } @@ -439,6 +447,13 @@ class Web3Service { })); } + async processBlockRange(fromBlock, toBlock, contract, topics) { + const newEvents = await Promise.all( + topics.map((topic) => contract.queryFilter(topic, fromBlock, toBlock)), + ); + return newEvents; + } + isOlderThan(timestamp, olderThanInMills) { if (!timestamp) return true; const timestampThirtyDaysInPast = new Date().getTime() - olderThanInMills; @@ -510,6 +525,14 @@ class Web3Service { ]); } + async getKnowledgeAssetOwner(assetContractAddress, tokenId) { + const assetStorageContractInstance = + this.assetStorageContracts[assetContractAddress.toString().toLowerCase()]; + if (!assetStorageContractInstance) throw Error('Unknown asset storage contract address'); + + return this.callContractFunction(assetStorageContractInstance, 'ownerOf', [tokenId]); + } + async getUnfinalizedState(tokenId) { return this.callContractFunction( this.UnfinalizedStateStorageContract, diff --git a/src/modules/network/implementation/libp2p-service.js b/src/modules/network/implementation/libp2p-service.js index 22f604a912..1b99a1c873 100644 --- a/src/modules/network/implementation/libp2p-service.js +++ b/src/modules/network/implementation/libp2p-service.js @@ -12,7 +12,6 @@ import { encode, decode } from 'it-length-prefixed'; import { create as _create, createFromPrivKey, createFromB58String } from 'peer-id'; import { InMemoryRateLimiter } from 'rolling-rate-limiter'; import toobusy from 'toobusy-js'; -import { v5 as uuidv5 } from 'uuid'; import { mkdir, writeFile, readFile, stat } from 'fs/promises'; import ip from 'ip'; import { TimeoutController } from 'timeout-abort-controller'; @@ -176,12 +175,12 @@ class Libp2pService { return this.node.multiaddrs; } - getProtocols(peerId) { - return this.node.peerStore.protoBook.get(peerId); + getProtocols(peerIdObject) { + return this.node.peerStore.protoBook.get(peerIdObject); } - getAddresses(peerId) { - return this.node.peerStore.addressBook.get(peerId); + getAddresses(peerIdObject) { + return this.node.peerStore.addressBook.get(peerIdObject); } getPeers() { @@ -197,82 +196,92 @@ class Libp2pService { this.node.handle(protocol, async (handlerProps) => { const { stream } = handlerProps; - const remotePeerId = handlerProps.connection.remotePeer.toB58String(); + const peerIdString = handlerProps.connection.remotePeer.toB58String(); const { message, valid, busy } = await this._readMessageFromStream( stream, this.isRequestValid.bind(this), - remotePeerId, + peerIdString, ); this.updateSessionStream( message.header.operationId, message.header.keywordUuid, - remotePeerId, + peerIdString, stream, ); if (!valid) { await this.sendMessageResponse( protocol, - remotePeerId, + peerIdString, NETWORK_MESSAGE_TYPES.RESPONSES.NACK, message.header.operationId, message.header.keywordUuid, { errorMessage: 'Invalid request message' }, ); + this.removeCachedSession( + message.header.operationId, + message.header.keywordUuid, + peerIdString, + ); } else if (busy) { await this.sendMessageResponse( protocol, - remotePeerId, + peerIdString, NETWORK_MESSAGE_TYPES.RESPONSES.BUSY, message.header.operationId, message.header.keywordUuid, {}, ); + this.removeCachedSession( + message.header.operationId, + message.header.keywordUuid, + peerIdString, + ); } else { this.logger.debug( - `Receiving message from ${remotePeerId} to ${this.config.id}: protocol: ${protocol}, messageType: ${message.header.messageType};`, + `Receiving message from ${peerIdString} to ${this.config.id}: protocol: ${protocol}, messageType: ${message.header.messageType};`, ); - await handler(message, remotePeerId); + await handler(message, peerIdString); } }); } - updateSessionStream(operationId, keywordUuid, remotePeerId, stream) { + updateSessionStream(operationId, keywordUuid, peerIdString, stream) { this.logger.trace( - `Storing new session stream for remotePeerId: ${remotePeerId} with operation id: ${operationId}`, + `Storing new session stream for remotePeerId: ${peerIdString} with operation id: ${operationId}`, ); - if (!this.sessions[remotePeerId]) { - this.sessions[remotePeerId] = { + if (!this.sessions[peerIdString]) { + this.sessions[peerIdString] = { [operationId]: { [keywordUuid]: { stream, }, }, }; - } else if (!this.sessions[remotePeerId][operationId]) { - this.sessions[remotePeerId][operationId] = { + } else if (!this.sessions[peerIdString][operationId]) { + this.sessions[peerIdString][operationId] = { [keywordUuid]: { stream, }, }; } else { - this.sessions[remotePeerId][operationId][keywordUuid] = { + this.sessions[peerIdString][operationId][keywordUuid] = { stream, }; } } - getSessionStream(operationId, keywordUuid, remotePeerId) { + getSessionStream(operationId, keywordUuid, peerIdString) { if ( - this.sessions[remotePeerId] && - this.sessions[remotePeerId][operationId] && - this.sessions[remotePeerId][operationId][keywordUuid] + this.sessions[peerIdString] && + this.sessions[peerIdString][operationId] && + this.sessions[peerIdString][operationId][keywordUuid] ) { this.logger.trace( - `Session found remotePeerId: ${remotePeerId}, operation id: ${operationId}`, + `Session found remotePeerId: ${peerIdString}, operation id: ${operationId}`, ); - return this.sessions[remotePeerId][operationId][keywordUuid].stream; + return this.sessions[peerIdString][operationId][keywordUuid].stream; } return null; } @@ -288,55 +297,57 @@ class Libp2pService { }; } - async sendMessage(protocol, peerId, messageType, operationId, keyword, message, timeout) { + async sendMessage( + protocol, + peerIdString, + messageType, + operationId, + keywordUuid, + message, + timeout, + ) { const nackMessage = { header: { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK }, data: { errorMessage: '', }, }; - const keywordUuid = uuidv5(keyword, uuidv5.URL); - - // const sessionStream = this.getSessionStream(operationId, remotePeerId.toB58String()); - // if (!sessionStream) { - // } else { - // stream = sessionStream; - // } - const remotePeerId = createFromB58String(peerId); + const peerIdObject = createFromB58String(peerIdString); - const publicIp = (this.getAddresses(remotePeerId) ?? []) + const publicIp = (this.getAddresses(peerIdObject) ?? []) .map((addr) => addr.multiaddr) .filter((addr) => addr.isThinWaistAddress()) .map((addr) => addr.toString().split('/')) .filter((splittedAddr) => !ip.isPrivate(splittedAddr[2]))[0]?.[2]; this.logger.trace( - `Dialing remotePeerId: ${remotePeerId.toB58String()} with public ip: ${publicIp}: protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}`, + `Dialing remotePeerId: ${peerIdString} with public ip: ${publicIp}: protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}`, ); let dialResult; let dialStart; let dialEnd; try { dialStart = Date.now(); - dialResult = await this.node.dialProtocol(remotePeerId, protocol); + dialResult = await this.node.dialProtocol(peerIdObject, protocol); dialEnd = Date.now(); } catch (error) { dialEnd = Date.now(); - nackMessage.data.errorMessage = `Unable to dial peer: ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ + nackMessage.data.errorMessage = `Unable to dial peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ dialEnd - dialStart } ms. Error: ${error.message}`; return nackMessage; } this.logger.trace( - `Created stream for peer: ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ + `Created stream for peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ dialEnd - dialStart } ms.`, ); + const { stream } = dialResult; - this.updateSessionStream(operationId, keywordUuid, remotePeerId.toB58String(), stream); + this.updateSessionStream(operationId, keywordUuid, peerIdString, stream); const streamMessage = this.createStreamMessage( message, @@ -346,7 +357,7 @@ class Libp2pService { ); this.logger.trace( - `Sending message to ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}`, + `Sending message to ${peerIdString}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}`, ); let sendMessageStart; @@ -357,27 +368,13 @@ class Libp2pService { sendMessageEnd = Date.now(); } catch (error) { sendMessageEnd = Date.now(); - nackMessage.data.errorMessage = `Unable to send message to peer: ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}, execution time: ${ + nackMessage.data.errorMessage = `Unable to send message to peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}, execution time: ${ sendMessageEnd - sendMessageStart } ms. Error: ${error.message}`; return nackMessage; } - // if (!this.sessions[remotePeerId.toB58String()]) { - // this.sessions[remotePeerId.toB58String()] = { - // [operationId]: { - // stream - // } - // } - // } else { - // this.sessions[remotePeerId.toB58String()][operationId] = { - // stream - // } - // } - // if (!this.sessions.sender[message.header.sessionId]) { - // this.sessions.sender[message.header.sessionId] = {}; - // } let readResponseStart; let readResponseEnd; let response; @@ -397,7 +394,7 @@ class Libp2pService { response = await this._readMessageFromStream( stream, this.isResponseValid.bind(this), - remotePeerId.toB58String(), + peerIdString, ); if (timeoutController.signal.aborted) { @@ -413,7 +410,7 @@ class Libp2pService { timeoutController.clear(); readResponseEnd = Date.now(); - nackMessage.data.errorMessage = `Unable to read response from peer ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, execution time: ${ + nackMessage.data.errorMessage = `Unable to read response from peer ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, execution time: ${ readResponseEnd - readResponseStart } ms. Error: ${error.message}`; @@ -421,7 +418,7 @@ class Libp2pService { } this.logger.trace( - `Receiving response from ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${ + `Receiving response from ${peerIdString}. protocol: ${protocol}, messageType: ${ response.message?.header?.messageType }, operationId: ${operationId}, execution time: ${ readResponseEnd - readResponseStart @@ -439,19 +436,19 @@ class Libp2pService { async sendMessageResponse( protocol, - remotePeerId, + peerIdString, messageType, operationId, keywordUuid, message, ) { this.logger.debug( - `Sending response from ${this.config.id} to ${remotePeerId}: protocol: ${protocol}, messageType: ${messageType};`, + `Sending response from ${this.config.id} to ${peerIdString}: protocol: ${protocol}, messageType: ${messageType};`, ); - const stream = this.getSessionStream(operationId, keywordUuid, remotePeerId); + const stream = this.getSessionStream(operationId, keywordUuid, peerIdString); if (!stream) { - throw Error(`Unable to find opened stream for remotePeerId: ${remotePeerId}`); + throw Error(`Unable to find opened stream for remotePeerId: ${peerIdString}`); } const response = this.createStreamMessage(message, operationId, keywordUuid, messageType); @@ -459,35 +456,6 @@ class Libp2pService { await this._sendMessageToStream(stream, response); } - // updateReceiverSession(header) { - // // if BUSY we expect same request, so don't update session - // if (header.messageType === constants.NETWORK_MESSAGE_TYPES.RESPONSES.BUSY) return; - // // if NACK we don't expect other requests, so delete session - // if (header.messageType === constants.NETWORK_MESSAGE_TYPES.RESPONSES.NACK) { - // if (header.sessionId) delete this.sessions.receiver[header.sessionId]; - // return; - // } - // - // // if session is new, initialise array of expected message types - // if (!this.sessions.receiver[header.sessionId].expectedMessageTypes) { - // this.sessions.receiver[header.sessionId].expectedMessageTypes = Object.keys( - // constants.NETWORK_MESSAGE_TYPES.REQUESTS, - // ); - // } - // - // // subroutine completed - // if (header.messageType === constants.NETWORK_MESSAGE_TYPES.RESPONSES.ACK) { - // // protocol operation completed, delete session - // if (this.sessions.receiver[header.sessionId].expectedMessageTypes.length <= 1) { - // this.removeSession(header.sessionId); - // } else { - // // operation not completed, update array of expected message types - // this.sessions.receiver[header.sessionId].expectedMessageTypes = - // this.sessions.receiver[header.sessionId].expectedMessageTypes.slice(1); - // } - // } - // } - async _sendMessageToStream(stream, message) { const stringifiedHeader = JSON.stringify(message.header); const stringifiedData = JSON.stringify(message.data); @@ -511,7 +479,7 @@ class Libp2pService { ); } - async _readMessageFromStream(stream, isMessageValid, remotePeerId) { + async _readMessageFromStream(stream, isMessageValid, peerIdString) { return pipe( // Read from the stream (the source) stream.source, @@ -520,11 +488,11 @@ class Libp2pService { // Turn buffers into strings (source) => map(source, (buf) => buf.toString()), // Sink function - (source) => this.readMessageSink(source, isMessageValid, remotePeerId), + (source) => this.readMessageSink(source, isMessageValid, peerIdString), ); } - async readMessageSink(source, isMessageValid, remotePeerId) { + async readMessageSink(source, isMessageValid, peerIdString) { const message = { header: { operationId: '', keywordUuid: '' }, data: {} }; // we expect first buffer to be header const stringifiedHeader = (await source.next()).value; @@ -536,7 +504,7 @@ class Libp2pService { message.header = JSON.parse(stringifiedHeader); // validate request / response - if (!(await isMessageValid(message.header, remotePeerId))) { + if (!(await isMessageValid(message.header, peerIdString))) { return { message, valid: false }; } @@ -558,9 +526,9 @@ class Libp2pService { return { message, valid: true, busy: false }; } - async isRequestValid(header, remotePeerId) { + async isRequestValid(header, peerIdString) { // filter spam requests - if (await this.limitRequest(header, remotePeerId)) return false; + if (await this.limitRequest(header, peerIdString)) return false; // header well formed if ( @@ -574,30 +542,15 @@ class Libp2pService { return true; } - return this.sessionExists(remotePeerId, header.operationId, header.keywordUuid); + return this.sessionExists(peerIdString, header.operationId, header.keywordUuid); } sessionExists() { return true; - // return this.sessions[remotePeerId.toB58String()] && this.sessions[remotePeerId.toB58String()][operationId]; } async isResponseValid() { return true; - // return ( - // header.operationId && - // header.messageType && - // this.sessions[remotePeerId][header.operationId] && - // Object.keys(constants.NETWORK_MESSAGE_TYPES.RESPONSES).includes(header.messageType) - // ); - } - - removeSession(sessionId) { - if (this.sessions.sender[sessionId]) { - delete this.sessions.sender[sessionId]; - } else if (this.sessions.receiver[sessionId]) { - delete this.sessions.receiver[sessionId]; - } } healthCheck() { @@ -607,36 +560,36 @@ class Libp2pService { return false; } - async limitRequest(header, remotePeerId) { + async limitRequest(header, peerIdString) { // if (header.sessionId && this.sessions.receiver[header.sessionId]) return false; - if (this.blackList[remotePeerId]) { + if (this.blackList[peerIdString]) { const remainingMinutes = Math.floor( NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES - - (Date.now() - this.blackList[remotePeerId]) / (1000 * 60), + (Date.now() - this.blackList[peerIdString]) / (1000 * 60), ); if (remainingMinutes > 0) { this.logger.debug( - `Blocking request from ${remotePeerId}. Node is blacklisted for ${remainingMinutes} minutes.`, + `Blocking request from ${peerIdString}. Node is blacklisted for ${remainingMinutes} minutes.`, ); return true; } - delete this.blackList[remotePeerId]; + delete this.blackList[peerIdString]; } - if (await this.rateLimiter.spamDetection.limit(remotePeerId)) { - this.blackList[remotePeerId] = Date.now(); + if (await this.rateLimiter.spamDetection.limit(peerIdString)) { + this.blackList[peerIdString] = Date.now(); this.logger.debug( - `Blocking request from ${remotePeerId}. Spammer detected and blacklisted for ${NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES} minutes.`, + `Blocking request from ${peerIdString}. Spammer detected and blacklisted for ${NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES} minutes.`, ); return true; } - if (await this.rateLimiter.basicRateLimiter.limit(remotePeerId)) { + if (await this.rateLimiter.basicRateLimiter.limit(peerIdString)) { this.logger.debug( - `Blocking request from ${remotePeerId}. Max number of requests exceeded.`, + `Blocking request from ${peerIdString}. Max number of requests exceeded.`, ); return true; @@ -674,6 +627,16 @@ class Libp2pService { async getPeerInfo(peerId) { return this.node.peerStore.get(createFromB58String(peerId)); } + + removeCachedSession(operationId, keywordUuid, peerIdString) { + if (this.sessions[peerIdString]?.[operationId]?.[keywordUuid]?.stream) { + this.sessions[peerIdString][operationId][keywordUuid].stream.close(); + delete this.sessions[peerIdString][operationId]; + this.logger.trace( + `Removed session for remotePeerId: ${peerIdString}, operationId: ${operationId}.`, + ); + } + } } export default Libp2pService; diff --git a/src/modules/network/network-module-manager.js b/src/modules/network/network-module-manager.js index 6ba254d664..758ca4244a 100644 --- a/src/modules/network/network-module-manager.js +++ b/src/modules/network/network-module-manager.js @@ -23,14 +23,22 @@ class NetworkModuleManager extends BaseModuleManager { } } - async sendMessage(protocol, remotePeerId, messageType, operationId, keyword, message, timeout) { + async sendMessage( + protocol, + remotePeerId, + messageType, + operationId, + keywordUuid, + message, + timeout, + ) { if (this.initialized) { return this.getImplementation().module.sendMessage( protocol, remotePeerId, messageType, operationId, - keyword, + keywordUuid, message, timeout, ); @@ -56,12 +64,6 @@ class NetworkModuleManager extends BaseModuleManager { } } - removeSession(sessionId) { - if (this.initialized) { - this.getImplementation().module.removeSession(sessionId); - } - } - getPeerId() { if (this.initialized) { return this.getImplementation().module.getPeerId(); @@ -91,6 +93,16 @@ class NetworkModuleManager extends BaseModuleManager { return this.getImplementation().module.getPeerInfo(peerId); } } + + removeCachedSession(operationId, keywordUuid, remotePeerId) { + if (this.initialized) { + this.getImplementation().module.removeCachedSession( + operationId, + keywordUuid, + remotePeerId, + ); + } + } } export default NetworkModuleManager; diff --git a/src/modules/repository/implementation/sequelize/models/blockchain-event.js b/src/modules/repository/implementation/sequelize/models/blockchain-event.js index c555dd1b8d..b15537849f 100644 --- a/src/modules/repository/implementation/sequelize/models/blockchain-event.js +++ b/src/modules/repository/implementation/sequelize/models/blockchain-event.js @@ -13,6 +13,8 @@ export default (sequelize, DataTypes) => { data: DataTypes.TEXT, block: DataTypes.INTEGER, processed: DataTypes.BOOLEAN, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js index 9d21770b4a..151f221ddc 100644 --- a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js @@ -55,6 +55,26 @@ class BlockchainEventRepository { }, ); } + + async removeEvents(ids) { + await this.model.destroy({ + where: { + id: { [Sequelize.Op.in]: ids }, + }, + }); + } + + async findProcessedEvents(timestamp, limit) { + return this.model.findAll({ + where: { + processed: true, + createdAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['createdAt', 'asc']], + raw: true, + limit, + }); + } } export default BlockchainEventRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/command-repository.js b/src/modules/repository/implementation/sequelize/repositories/command-repository.js index b272f1b71e..44c08b6bed 100644 --- a/src/modules/repository/implementation/sequelize/repositories/command-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/command-repository.js @@ -1,4 +1,5 @@ import Sequelize from 'sequelize'; +import { COMMAND_STATUS } from '../../../../../constants/constants.js'; class CommandRepository { constructor(models) { @@ -41,14 +42,32 @@ class CommandRepository { }); } - async removeFinalizedCommands(finalizedStatuses) { + async removeCommands(ids) { await this.model.destroy({ where: { - status: { [Sequelize.Op.in]: finalizedStatuses }, - startedAt: { [Sequelize.Op.lte]: Date.now() }, + id: { [Sequelize.Op.in]: ids }, }, }); } + + async findFinalizedCommands(timestamp, limit) { + return this.model.findAll({ + where: { + status: { + [Sequelize.Op.in]: [ + COMMAND_STATUS.COMPLETED, + COMMAND_STATUS.FAILED, + COMMAND_STATUS.EXPIRED, + COMMAND_STATUS.UNKNOWN, + ], + }, + startedAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['startedAt', 'asc']], + raw: true, + limit, + }); + } } export default CommandRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-repository.js b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js index 63efad192b..e0634aaccb 100644 --- a/src/modules/repository/implementation/sequelize/repositories/operation-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js @@ -1,3 +1,5 @@ +import { Sequelize } from 'sequelize'; + class OperationRepository { constructor(models) { this.sequelize = models.sequelize; @@ -11,6 +13,25 @@ class OperationRepository { }); } + async removeOperationRecords(operation, ids) { + return this.models[operation].destroy({ + where: { + id: { [Sequelize.Op.in]: ids }, + }, + }); + } + + async findProcessedOperations(operation, timestamp, limit) { + return this.models[`${operation}`].findAll({ + where: { + createdAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['createdAt', 'asc']], + raw: true, + limit, + }); + } + async getOperationStatus(operation, operationId) { return this.models[operation].findOne({ attributes: ['status'], diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-response.js b/src/modules/repository/implementation/sequelize/repositories/operation-response.js index 87854a2b2e..7c55f5d139 100644 --- a/src/modules/repository/implementation/sequelize/repositories/operation-response.js +++ b/src/modules/repository/implementation/sequelize/repositories/operation-response.js @@ -1,3 +1,5 @@ +import Sequelize from 'sequelize'; + class OperationResponseRepository { constructor(models) { this.sequelize = models.sequelize; @@ -25,6 +27,25 @@ class OperationResponseRepository { }, }); } + + async findProcessedOperationResponse(timestamp, limit, operation) { + return this.models[`${operation}_response`].findAll({ + where: { + createdAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['createdAt', 'asc']], + raw: true, + limit, + }); + } + + async removeOperationResponse(ids, operation) { + await this.models[`${operation}_response`].destroy({ + where: { + id: { [Sequelize.Op.in]: ids }, + }, + }); + } } export default OperationResponseRepository; diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 688cebe9de..fd1cc4617f 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -56,8 +56,12 @@ class RepositoryModuleManager extends BaseModuleManager { return this.getRepository('command').getCommandWithId(id); } - async removeFinalizedCommands(finalizedStatuses) { - return this.getRepository('command').removeFinalizedCommands(finalizedStatuses); + async removeCommands(ids) { + return this.getRepository('command').removeCommands(ids); + } + + async findFinalizedCommands(timestamp, limit) { + return this.getRepository('command').findFinalizedCommands(timestamp, limit); } async createOperationIdRecord(handlerData) { @@ -87,6 +91,14 @@ class RepositoryModuleManager extends BaseModuleManager { ); } + async removeOperationRecords(operation, ids) { + return this.getRepository('operation').removeOperationRecords(operation, ids); + } + + async findProcessedOperations(operation, timestamp, limit) { + return this.getRepository('operation').findProcessedOperations(operation, timestamp, limit); + } + async getOperationStatus(operation, operationId) { return this.getRepository('operation').getOperationStatus(operation, operationId); } @@ -116,6 +128,18 @@ class RepositoryModuleManager extends BaseModuleManager { ); } + async findProcessedOperationResponse(timestamp, limit, operation) { + return this.getRepository('operation_response').findProcessedOperationResponse( + timestamp, + limit, + operation, + ); + } + + async removeOperationResponse(ids, operation) { + return this.getRepository('operation_response').removeOperationResponse(ids, operation); + } + // Sharding Table async createManyPeerRecords(peers) { return this.getRepository('shard').createManyPeerRecords(peers); @@ -238,6 +262,14 @@ class RepositoryModuleManager extends BaseModuleManager { return this.getRepository('blockchain_event').removeBlockchainEvents(contract); } + async removeEvents(ids) { + return this.getRepository('blockchain_event').removeEvents(ids); + } + + async findProcessedEvents(timestamp, limit) { + return this.getRepository('blockchain_event').findProcessedEvents(timestamp, limit); + } + async removeLastCheckedBlockForContract(contract) { return this.getRepository('blockchain').removeLastCheckedBlockForContract(contract); } diff --git a/src/service/archive-service.js b/src/service/archive-service.js new file mode 100644 index 0000000000..1fda755ae0 --- /dev/null +++ b/src/service/archive-service.js @@ -0,0 +1,19 @@ +class ArchiveService { + constructor(ctx) { + this.logger = ctx.logger; + this.fileService = ctx.fileService; + } + + async archiveData(archiveFolderName, archiveFileName, data) { + const archiveFolderPath = this.fileService.getArchiveFolderPath(archiveFolderName); + this.logger.debug( + `Archiving data on path: ${archiveFolderPath} with archive name: ${archiveFileName}`, + ); + await this.fileService.writeContentsToFile( + archiveFolderPath, + archiveFileName, + JSON.stringify(data), + ); + } +} +export default ArchiveService; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 2ce399559e..6c3c0721ce 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -11,10 +11,7 @@ import { const MAXIMUM_FETCH_EVENTS_FAILED_COUNT = 5; const fetchEventsFailedCount = {}; -const eventNames = []; -Object.keys(CONTRACT_EVENTS).forEach((contractName) => { - eventNames.push(...Object.values(CONTRACT_EVENTS[contractName])); -}); +const eventNames = Object.values(CONTRACT_EVENTS).flatMap((e) => e); class BlockchainEventListenerService { constructor(ctx) { @@ -52,24 +49,46 @@ class BlockchainEventListenerService { const currentBlock = await this.blockchainModuleManager.getBlockNumber(); const syncContractEventsPromises = [ - this.getContractEvents(blockchainId, CONTRACTS.SHARDING_TABLE_CONTRACT, currentBlock), - this.getContractEvents(blockchainId, CONTRACTS.STAKING_CONTRACT, currentBlock), - this.getContractEvents(blockchainId, CONTRACTS.PROFILE_CONTRACT, currentBlock), + this.getContractEvents( + blockchainId, + CONTRACTS.SHARDING_TABLE_CONTRACT, + currentBlock, + CONTRACT_EVENTS.SHARDING_TABLE, + ), + this.getContractEvents( + blockchainId, + CONTRACTS.STAKING_CONTRACT, + currentBlock, + CONTRACT_EVENTS.STAKING, + ), + this.getContractEvents( + blockchainId, + CONTRACTS.PROFILE_CONTRACT, + currentBlock, + CONTRACT_EVENTS.PROFILE, + ), this.getContractEvents( blockchainId, CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, currentBlock, + CONTRACT_EVENTS.COMMIT_MANAGER_V1, ), this.getContractEvents( blockchainId, CONTRACTS.SERVICE_AGREEMENT_V1_CONTRACT, currentBlock, + CONTRACT_EVENTS.SERVICE_AGREEMENT_V1, ), ]; if (!devEnvironment) { syncContractEventsPromises.push( - this.getContractEvents(blockchainId, CONTRACTS.HUB_CONTRACT, currentBlock), + this.getContractEvents( + blockchainId, + CONTRACTS.HUB_CONTRACT, + currentBlock, + CONTRACT_EVENTS.HUB, + ), ); } const contractEvents = await Promise.all(syncContractEventsPromises); @@ -118,7 +137,7 @@ class BlockchainEventListenerService { }, eventFetchInterval); } - async getContractEvents(blockchainId, contractName, currentBlock) { + async getContractEvents(blockchainId, contractName, currentBlock, eventsToFilter) { const lastCheckedBlockObject = await this.repositoryModuleManager.getLastCheckedBlock( blockchainId, contractName, @@ -127,6 +146,7 @@ class BlockchainEventListenerService { const events = await this.blockchainModuleManager.getAllPastEvents( blockchainId, contractName, + eventsToFilter, lastCheckedBlockObject?.lastCheckedBlock ?? 0, lastCheckedBlockObject?.lastCheckedTimestamp ?? 0, currentBlock, diff --git a/src/service/file-service.js b/src/service/file-service.js index b428bf18fd..aa1c5be91b 100644 --- a/src/service/file-service.js +++ b/src/service/file-service.js @@ -1,17 +1,19 @@ import path from 'path'; -import { mkdir, writeFile, readFile, unlink, stat, readdir } from 'fs/promises'; +import { mkdir, writeFile, readFile, unlink, stat, readdir, rm } from 'fs/promises'; import appRootPath from 'app-root-path'; const MIGRATION_FOLDER_NAME = 'migrations'; +const ARCHIVE_FOLDER_NAME = 'archive'; + class FileService { constructor(ctx) { this.config = ctx.config; this.logger = ctx.logger; } - getFileExtension(fileName) { - return path.extname(fileName).toLowerCase(); + getFileExtension(filePath) { + return path.extname(filePath).toLowerCase(); } /** @@ -23,7 +25,7 @@ class FileService { */ async writeContentsToFile(directory, filename, data, log = true) { if (log) { - this.logger.debug(`Saving file with name: ${filename} in directory: ${directory}`); + this.logger.debug(`Saving file with name: ${filename} in the directory: ${directory}`); } await mkdir(directory, { recursive: true }); const fullpath = path.join(directory, filename); @@ -31,56 +33,75 @@ class FileService { return fullpath; } - readFileOnPath(filePath) { - return this._readFile(filePath, false); - } - async readDirectory(dirPath) { - return readdir(dirPath); + this.logger.debug(`Reading folder at path: ${dirPath}`); + try { + return readdir(dirPath); + } catch (error) { + if (error.code === 'ENOENT') { + throw Error(`Folder not found at path: ${dirPath}`); + } + throw error; + } } async stat(filePath) { return stat(filePath); } - /** - * Loads JSON data from file - * @returns {Promise} - * @private - */ - loadJsonFromFile(filePath) { - return this._readFile(filePath, true); - } - - async fileExists(filePath) { + async pathExists(fileOrDirPath) { try { - await stat(filePath); + await stat(fileOrDirPath); return true; - } catch (e) { - return false; + } catch (error) { + if (error.code === 'ENOENT') { + return false; + } + throw error; } } - async _readFile(filePath, convertToJSON = false) { - this.logger.debug( - `Reading file on path: ${filePath}, converting to json: ${convertToJSON}`, - ); + async readFile(filePath, convertToJSON = false) { + this.logger.debug(`Reading file: ${filePath}, converting to json: ${convertToJSON}`); try { const data = await readFile(filePath); return convertToJSON ? JSON.parse(data) : data.toString(); - } catch (e) { - throw Error(`File not found on path: ${filePath}`); + } catch (error) { + if (error.code === 'ENOENT') { + throw Error(`File not found at path: ${filePath}`); + } + throw error; } } async removeFile(filePath) { - if (await this.fileExists(filePath)) { - this.logger.trace(`Removing file on path: ${filePath}`); + this.logger.trace(`Removing file at path: ${filePath}`); + + try { await unlink(filePath); return true; + } catch (error) { + if (error.code === 'ENOENT') { + this.logger.debug(`File not found at path: ${filePath}`); + return false; + } + throw error; + } + } + + async removeFolder(folderPath) { + // this.logger.trace(`Removing folder at path: ${folderPath}`); + + try { + await rm(folderPath, { recursive: true }); + return true; + } catch (error) { + if (error.code === 'ENOENT') { + this.logger.debug(`Folder not found at path: ${folderPath}`); + return false; + } + throw error; } - this.logger.debug(`File not found on path: ${filePath}`); - return false; } getDataFolderPath() { @@ -106,19 +127,37 @@ class FileService { return path.join(this.getOperationIdCachePath(), operationId); } - getPendingStorageFileName(blockchain, contract, tokenId) { - return `${blockchain.toLowerCase()}:${contract.toLowerCase()}:${tokenId}`; - } - getPendingStorageCachePath(repository) { return path.join(this.getDataFolderPath(), 'pending_storage_cache', repository); } - getPendingStorageDocumentPath(repository, blockchain, contract, tokenId) { + getPendingStorageFolderPath(repository, blockchain, contract, tokenId) { return path.join( this.getPendingStorageCachePath(repository), - this.getPendingStorageFileName(blockchain, contract, tokenId), + `${blockchain.toLowerCase()}:${contract.toLowerCase()}:${tokenId}`, + ); + } + + async getPendingStorageDocumentPath(repository, blockchain, contract, tokenId, assertionId) { + const pendingStorageFolder = this.getPendingStorageFolderPath( + repository, + blockchain, + contract, + tokenId, ); + + let pendingStorageFileName; + if (assertionId === undefined) { + [pendingStorageFileName] = await this.readDirectory(pendingStorageFolder); + } else { + pendingStorageFileName = assertionId; + } + + return path.join(pendingStorageFolder, pendingStorageFileName); + } + + getArchiveFolderPath(subFolder) { + return path.join(this.getDataFolderPath(), ARCHIVE_FOLDER_NAME, subFolder); } } diff --git a/src/service/operation-id-service.js b/src/service/operation-id-service.js index eb727cbe79..3e714c2a75 100644 --- a/src/service/operation-id-service.js +++ b/src/service/operation-id-service.js @@ -1,4 +1,4 @@ -import { validate } from 'uuid'; +import { validate, v4 as uuidv4 } from 'uuid'; import path from 'path'; class OperationIdService { @@ -11,6 +11,10 @@ class OperationIdService { this.memoryCachedHandlersData = {}; } + generateId() { + return uuidv4(); + } + async generateOperationId(status) { const operationIdObject = await this.repositoryModuleManager.createOperationIdRecord({ status, @@ -109,8 +113,8 @@ class OperationIdService { this.logger.debug(`Reading operation id: ${operationId} cached data from file`); const documentPath = this.fileService.getOperationIdDocumentPath(operationId); let data; - if (await this.fileService.fileExists(documentPath)) { - data = await this.fileService.loadJsonFromFile(documentPath); + if (await this.fileService.pathExists(documentPath)) { + data = await this.fileService.readFile(documentPath, true); } return data; } @@ -140,16 +144,17 @@ class OperationIdService { return deleted; } - async removeExpiredOperationIdFileCache(expiredTimeout) { + async removeExpiredOperationIdFileCache(expiredTimeout, batchSize) { const cacheFolderPath = this.fileService.getOperationIdCachePath(); - const cacheFolderExists = await this.fileService.fileExists(cacheFolderPath); + const cacheFolderExists = await this.fileService.pathExists(cacheFolderPath); if (!cacheFolderExists) { return; } const fileList = await this.fileService.readDirectory(cacheFolderPath); + + const now = new Date(); const deleteFile = async (fileName) => { const filePath = path.join(cacheFolderPath, fileName); - const now = new Date(); const createdDate = (await this.fileService.stat(filePath)).mtime; if (createdDate.getTime() + expiredTimeout < now.getTime()) { await this.fileService.removeFile(filePath); @@ -157,8 +162,17 @@ class OperationIdService { } return false; }; - const deleted = await Promise.all(fileList.map((fileName) => deleteFile(fileName))); - return deleted.filter((x) => x).length; + let totalDeleted = 0; + for (let i = 0; i < fileList.length; i += batchSize) { + const batch = fileList.slice(i, i + batchSize); + // eslint-disable-next-line no-await-in-loop + const deletionResults = await Promise.allSettled(batch.map(deleteFile)); + totalDeleted += deletionResults.filter( + (result) => result.status === 'fulfilled' && result.value, + ).length; + } + + return totalDeleted; } } diff --git a/src/service/operation-service.js b/src/service/operation-service.js index 8f97483051..13f4d64285 100644 --- a/src/service/operation-service.js +++ b/src/service/operation-service.js @@ -68,7 +68,9 @@ class OperationService { OPERATION_STATUS.COMPLETED, ); - await this.operationIdService.cacheOperationIdData(operationId, responseData); + if (responseData != null) { + await this.operationIdService.cacheOperationIdData(operationId, responseData); + } for (const status of endStatuses) { // eslint-disable-next-line no-await-in-loop diff --git a/src/service/pending-storage-service.js b/src/service/pending-storage-service.js index 700b3d3456..a882ce40af 100644 --- a/src/service/pending-storage-service.js +++ b/src/service/pending-storage-service.js @@ -1,28 +1,35 @@ class PendingStorageService { constructor(ctx) { this.logger = ctx.logger; - this.fileService = ctx.fileService; this.ualService = ctx.ualService; } - async cacheAssertion(repository, blockchain, contract, tokenId, assertion, operationId) { + async cacheAssertion( + repository, + blockchain, + contract, + tokenId, + assertionId, + assertion, + operationId, + ) { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.debug( - `Caching assertion for ual: ${ual}, operation id: ${operationId} in file in ${repository} pending storage`, + `Caching ${assertionId} assertion for ual: ${ual}, operation id: ${operationId} in file in ${repository} pending storage`, ); - const documentPath = this.fileService.getPendingStorageCachePath(repository); - const documentName = this.fileService.getPendingStorageFileName( + const pendingStorageFolderPath = this.fileService.getPendingStorageFolderPath( + repository, blockchain, contract, tokenId, ); await this.fileService.writeContentsToFile( - documentPath, - documentName, + pendingStorageFolderPath, + assertionId, JSON.stringify(assertion), ); } @@ -33,19 +40,20 @@ class PendingStorageService { this.logger.debug( `Reading cached assertion for ual: ${ual}, operation id: ${operationId} from file in ${repository} pending storage`, ); + try { + const documentPath = await this.fileService.getPendingStorageDocumentPath( + repository, + blockchain, + contract, + tokenId, + ); - const documentPath = this.fileService.getPendingStorageDocumentPath( - repository, - blockchain, - contract, - tokenId, - ); - let data; - if (await this.fileService.fileExists(documentPath)) { - data = await this.fileService.loadJsonFromFile(documentPath); + const data = await this.fileService.readFile(documentPath, true); + return data; + } catch (error) { + this.logger.debug('Assertion not found in pending storage'); + return null; } - - return data; } async removeCachedAssertion(repository, blockchain, contract, tokenId, operationId) { @@ -55,26 +63,31 @@ class PendingStorageService { `Removing cached assertion for ual: ${ual} operation id: ${operationId} from file in ${repository} pending storage`, ); - const documentPath = this.fileService.getPendingStorageDocumentPath( + const pendingStorageFolderPath = this.fileService.getPendingStorageFolderPath( repository, blockchain, contract, tokenId, ); - await this.fileService.removeFile(documentPath); + await this.fileService.removeFolder(pendingStorageFolderPath); } - async assertionExists(repository, blockchain, contract, tokenId) { - const documentPath = this.fileService.getPendingStorageDocumentPath( - repository, - blockchain, - contract, - tokenId, - ); - this.logger.trace( - `Checking if assertion exists in pending storage on path: ${documentPath}`, - ); - return this.fileService.fileExists(documentPath); + async assetHasPendingState(repository, blockchain, contract, tokenId, assertionId) { + try { + const documentPath = await this.fileService.getPendingStorageDocumentPath( + repository, + blockchain, + contract, + tokenId, + assertionId, + ); + this.logger.trace( + `Checking if assertion exists in pending storage at path: ${documentPath}`, + ); + return this.fileService.pathExists(documentPath); + } catch (error) { + return false; + } } } diff --git a/src/service/publish-service.js b/src/service/publish-service.js index 96d2feb40f..12b16edbf2 100644 --- a/src/service/publish-service.js +++ b/src/service/publish-service.js @@ -71,7 +71,7 @@ class PublishService extends OperationService { } } if (allCompleted) { - await this.markOperationAsCompleted(operationId, {}, this.completedStatuses); + await this.markOperationAsCompleted(operationId, null, this.completedStatuses); this.logResponsesSummary(completedNumber, failedNumber); this.logger.info( `${this.operationName} with operation id: ${operationId} with status: ${ diff --git a/src/service/update-service.js b/src/service/update-service.js index bc8ee70d37..758e4de96d 100644 --- a/src/service/update-service.js +++ b/src/service/update-service.js @@ -71,7 +71,7 @@ class UpdateService extends OperationService { } } if (allCompleted) { - await this.markOperationAsCompleted(operationId, {}, this.completedStatuses); + await this.markOperationAsCompleted(operationId, null, this.completedStatuses); this.logResponsesSummary(completedNumber, failedNumber); this.logger.info( `${this.operationName} with operation id: ${operationId} with status: ${ diff --git a/src/service/validation-service.js b/src/service/validation-service.js index ceaa96f861..d74700b897 100644 --- a/src/service/validation-service.js +++ b/src/service/validation-service.js @@ -7,6 +7,25 @@ class ValidationService { this.blockchainModuleManager = ctx.blockchainModuleManager; } + async validateUal(blockchain, contract, tokenId) { + this.logger.info( + `Validating UAL: did:dkg:${blockchain.toLowerCase()}/${contract.toLowerCase()}/${tokenId}`, + ); + + let isValid = true; + try { + await this.blockchainModuleManager.getKnowledgeAssetOwner( + blockchain, + contract, + tokenId, + ); + } catch (err) { + isValid = false; + } + + return isValid; + } + async validateAssertion(assertionId, blockchain, assertion) { this.logger.info(`Validating assertionId: ${assertionId}`); diff --git a/test/bdd/features/get-errors.feature b/test/bdd/features/get-errors.feature index 9ec5db4906..27b8516da3 100644 --- a/test/bdd/features/get-errors.feature +++ b/test/bdd/features/get-errors.feature @@ -4,23 +4,44 @@ Feature: Get errors test And 1 bootstrap is running @get-errors - Scenario: Getting non existent UAL + Scenario: Getting non-existent UAL + Given I setup 4 nodes + And I wait for 2 seconds + + When I call Get directly on the node 1 with nonExistentUAL + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: GetRouteError + + @get-errors + Scenario: Getting invalid UAL Given I setup 4 nodes And I wait for 2 seconds - And I call get directly to ot-node 1 with nonExistentUAL - And I wait for last resolve to finalize - Then Last Get operation finished with status: GetAssertionIdError - #@get-errors - #Scenario: GET operation result on a node with minimum replication factor greater than the number of nodes - #Given I setup 4 nodes - #And I wait for 2 seconds - #And I call publish on node 1 with validAssertion - #Then Last PUBLISH operation finished with status: COMPLETED - #When I setup node 5 with minimumAckResponses.get set to 10 - #And I wait for 2 seconds - #And I get operation result from node 5 for last published assertion - #And I wait for last resolve to finalize - #Then Last GET operation finished with status: GetNetworkError + When I call Get directly on the node 1 with invalidUAL + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: GetRouteError + @get-errors + Scenario: Getting non-existent state + Given I setup 4 nodes + And I set R0 to be 1 + And I set R1 to be 2 + And I wait for 2 seconds + + When I call Publish on the node 1 with validAssertion + And I wait for latest Publish to finalize + And I call Get directly on the node 1 with nonExistentState + Then It should fail with status code 400 + + @get-errors + Scenario: Getting invalid state hash + Given I setup 4 nodes + And I set R0 to be 1 + And I set R1 to be 2 + And I wait for 2 seconds + When I call Publish on the node 1 with validAssertion + And I wait for latest Publish to finalize + And I call Get directly on the node 1 with invalidStateHash + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: GetAssertionIdError diff --git a/test/bdd/features/get.feature b/test/bdd/features/get.feature new file mode 100644 index 0000000000..c6e994b212 --- /dev/null +++ b/test/bdd/features/get.feature @@ -0,0 +1,76 @@ +Feature: Get asset states test + Background: Setup local blockchain, bootstraps and nodes + Given the blockchain is set up + And 1 bootstrap is running + + @release + Scenario: Get first state of the updated knowledge asset + Given I set R0 to be 1 + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Get directly on the node 4 with validGetFirstStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + @release + Scenario: Get latest state of the updated knowledge asset + Given I set R0 to be 1 + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Get directly on the node 4 with validGetUpdatedStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + @release + Scenario: Get all states of the knowledge asset that is updated 2 times + Given I set R0 to be 1 + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_2 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Get directly on the node 4 with getFirstStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + When I call Get directly on the node 4 with getSecondStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + When I call Get directly on the node 4 with getThirdStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED diff --git a/test/bdd/features/publish-errors.feature b/test/bdd/features/publish-errors.feature index bf6754594c..47c62187fe 100644 --- a/test/bdd/features/publish-errors.feature +++ b/test/bdd/features/publish-errors.feature @@ -3,26 +3,20 @@ Feature: Publish errors test Given the blockchain is set up And 1 bootstrap is running - #@publish-errors - #Scenario: Publish on a node with invalid data path - #Given I setup 3 nodes - #And I setup node 4 with appDataPath set to \0 - #And I wait for 2 seconds - #And I call publish on node 4 with validAssertion - #Then Last PUBLISH operation finished with status: PublishRouteError - @publish-errors Scenario: Publish on a node with minimum replication factor greater than the number of nodes Given I setup 1 nodes - And I call publish on node 1 with validAssertion - Then Last Publish operation finished with status: PublishStartError + And I wait for 2 seconds + + When I call Publish on the node 1 with validAssertion + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: PublishStartError @publish-errors - Scenario: Publish an asset directly on the node + Scenario: Publish a knowledge asset directly on the node Given I setup 1 nodes - And I call publish on ot-node 1 directly with validPublishRequestBody - And I wait for last publish to finalize - Then Last Publish operation finished with status: ValidateAssetError -# -# + And I wait for 2 seconds + When I call Publish directly on the node 1 with validPublishRequestBody + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: ValidateAssetError diff --git a/test/bdd/features/publish.feature b/test/bdd/features/publish.feature index b38832ad56..0e28c9c9ed 100644 --- a/test/bdd/features/publish.feature +++ b/test/bdd/features/publish.feature @@ -6,9 +6,10 @@ Feature: Release related tests @release Scenario: Publishing a valid assertion Given I set R0 to be 1 - Given I set R1 to be 2 - Given I setup 4 nodes - And I wait for 10 seconds + And I set R1 to be 2 + And I setup 4 nodes + And I wait for 2 seconds - When I call publish on node 4 with validAssertion - Then Last Publish operation finished with status: COMPLETED + When I call Publish on the node 4 with validAssertion + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED diff --git a/test/bdd/features/update-errors.feature b/test/bdd/features/update-errors.feature index 794ea018df..6633eca3f2 100644 --- a/test/bdd/features/update-errors.feature +++ b/test/bdd/features/update-errors.feature @@ -4,9 +4,11 @@ Feature: Update errors test And 1 bootstrap is running @update-errors - Scenario: Update asset that was not previously published + Scenario: Update knowledge asset that was not previously published Given I setup 1 node - And I call update on ot-node 1 directly with validUpdateRequestBody - And I wait for last update to finalize - Then Last Update operation finished with status: ValidateAssetError + And I wait for 2 seconds + + When I call Update directly on the node 1 with validUpdateRequestBody + And I wait for latest Update to finalize + Then Latest Update operation finished with status: ValidateAssetError diff --git a/test/bdd/features/update.feature b/test/bdd/features/update.feature index 748c48013a..a70fd206ca 100644 --- a/test/bdd/features/update.feature +++ b/test/bdd/features/update.feature @@ -4,12 +4,17 @@ Feature: Update asset test And 1 bootstrap is running @release - Scenario: Update an existing asset + Scenario: Update an existing knowledge asset Given I set R0 to be 1 - Given I set R1 to be 2 - Given I setup 4 nodes - And I wait for 10 seconds - When I call publish on node 4 with validPublish_1ForValidUpdate_1 - Then Last Publish operation finished with status: COMPLETED - Given I call update on node 4 for last publish UAL with validUpdate_1 - When Last Update operation finished with status: COMPLETED + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED diff --git a/test/bdd/steps/api/datasets/assertions.json b/test/bdd/steps/api/datasets/assertions.json index 9f5115b082..57c923b3c1 100644 --- a/test/bdd/steps/api/datasets/assertions.json +++ b/test/bdd/steps/api/datasets/assertions.json @@ -61,5 +61,20 @@ "@id": "uuid:Nis" } } + }, + "validUpdate_2": { + "public": { + "@context": [ + "https://schema.org" + ], + "@id": "uuid:3", + "company": "TL", + "user": { + "@id": "uuid:user:3" + }, + "city": { + "@id": "uuid:Funchal" + } + } } } diff --git a/test/bdd/steps/api/datasets/requests.json b/test/bdd/steps/api/datasets/requests.json index 66e7e99bfc..be7258046b 100644 --- a/test/bdd/steps/api/datasets/requests.json +++ b/test/bdd/steps/api/datasets/requests.json @@ -9,9 +9,27 @@ "_:c14n0 ." ], "blockchain": "hardhat", - "contract": "0x5FbDB2315678afecb367f032d93F642f64180aa3", - "tokenId": 0, - "hashFunctionId": 1 + "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", + "tokenId": 0 + }, + "validGetFirstStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" + }, + "validGetUpdatedStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0" + }, + "getFirstStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" + }, + "getSecondStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584f" + }, + "getThirdStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0x759c285786b95622dad67a6be857a4eec3d9ba0caec991ed4297629ae6abbc0d" }, "blockchainNotDefinedRequestBody": { "publishType": "asset", @@ -24,11 +42,22 @@ "_:c14n0 ." ], "blockchain": null, - "contract": "0x791ee543738B997B7A125bc849005B62aFD35578", + "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", "tokenId": 0 }, "nonExistentUAL": { - "id": "did:ganache:0x791ee543738B997B7A125bc849005B62aFD35578/1" + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/1" + }, + "invalidUAL": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F/52b28595d31B441D079B2Ca07/1" + }, + "nonExistentState": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": -1 + }, + "invalidStateHash": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584e" }, "validUpdateRequestBody": { "assertionId": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584f", @@ -38,8 +67,7 @@ " ." ], "blockchain": "hardhat", - "contract": "0xb7278A61aa25c888815aFC32Ad3cC52fF24fE575", - "tokenId": 0, - "hashFunctionId": 1 + "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", + "tokenId": 0 } } diff --git a/test/bdd/steps/api/get.mjs b/test/bdd/steps/api/get.mjs new file mode 100644 index 0000000000..47ea8df7a1 --- /dev/null +++ b/test/bdd/steps/api/get.mjs @@ -0,0 +1,105 @@ +import { Then, When } from '@cucumber/cucumber'; +import { expect, assert } from 'chai'; +import { readFile } from 'fs/promises'; +import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; + +const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests.json')); + +const httpApiHelper = new HttpApiHelper(); + +When( + /^I call Get on the node (\d+) for state index (\d+)/, + { timeout: 120000 }, + async function get(node, stateIndex) { + this.logger.log(`I call get route on the node ${node} for state index ${stateIndex}.`); + + const { UAL } = this.state.latestUpdateData; + const result = await this.state.nodes[node - 1].client + .getHistorical(UAL, stateIndex) + .catch((error) => { + assert.fail(`Error while trying to update assertion. ${error}`); + }); + const { operationId } = result.operation; + this.state.latestUpdateData = { + nodeId: node - 1, + operationId, + }; + }, +); + +When( + /^I call Get directly on the node (\d+) with ([^"]*)/, + { timeout: 30000 }, + async function getFromNode(node, requestName) { + this.logger.log(`I call get directly on the node ${node}`); + expect( + !!requests[requestName], + `Request body with name: ${requestName} not found!`, + ).to.be.equal(true); + const requestBody = requests[requestName]; + + try { + const result = await httpApiHelper.get(this.state.nodes[node - 1].nodeRpcUrl, requestBody); + const { operationId } = result.data; + this.state.latestGetData = { + nodeId: node - 1, + operationId, + }; + } catch (error) { + this.state.latestError = error; + } + }, +); + +Then( + /^It should fail with status code (\d+)/, + function checkLatestError(expectedStatusCode) { + const expectedStatusCodeInt = parseInt(expectedStatusCode, 10); + assert( + this.state.latestError, + 'No error occurred' + ); + assert( + this.state.latestError.statusCode, + 'No status code in error' + ); + assert( + this.state.latestError.statusCode === expectedStatusCodeInt, + `Expected request to fail with status code ${expectedStatusCodeInt}, but it failed with another code.` + ); + }, +); + +When('I wait for latest Get to finalize', { timeout: 80000 }, async function getFinalize() { + this.logger.log('I wait for latest get to finalize'); + expect( + !!this.state.latestGetData, + 'Latest get data is undefined. Get was not started.', + ).to.be.equal(true); + const getData = this.state.latestGetData; + let retryCount = 0; + const maxRetryCount = 5; + for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { + this.logger.log( + `Getting get result for operation id: ${getData.operationId} on the node: ${getData.nodeId}`, + ); + // eslint-disable-next-line no-await-in-loop + const getResult = await httpApiHelper.getOperationResult( + this.state.nodes[getData.nodeId].nodeRpcUrl, + 'get', + getData.operationId, + ); + this.logger.log(`Operation status: ${getResult.data.status}`); + if (['COMPLETED', 'FAILED'].includes(getResult.data.status)) { + this.state.latestGetData.result = getResult; + this.state.latestGetData.status = getResult.data.status; + this.state.latestGetData.errorType = getResult.data.data?.errorType; + break; + } + if (retryCount === maxRetryCount - 1) { + assert.fail('Unable to fetch get result'); + } + // eslint-disable-next-line no-await-in-loop + await setTimeout(4000); + } +}); diff --git a/test/bdd/steps/api/info.mjs b/test/bdd/steps/api/info.mjs index 039a834506..017b4be049 100644 --- a/test/bdd/steps/api/info.mjs +++ b/test/bdd/steps/api/info.mjs @@ -3,9 +3,9 @@ import assert from 'assert'; let info = {}; -When(/^I call info route on node (\d+)/, { timeout: 120000 }, async function infoRouteCall(node) { +When(/^I call Info route on the node (\d+)/, { timeout: 120000 }, async function infoRouteCall(node) { // todo validate node number - this.logger.log('I call info route on node: ', node); + this.logger.log(`I call info route on the node ${node}`); info = await this.state.nodes[node - 1].client.info(); }); diff --git a/test/bdd/steps/api/publish.mjs b/test/bdd/steps/api/publish.mjs index ec7dcab97e..87a3d5ab52 100644 --- a/test/bdd/steps/api/publish.mjs +++ b/test/bdd/steps/api/publish.mjs @@ -1,4 +1,4 @@ -import { When, Given } from '@cucumber/cucumber'; +import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { setTimeout } from 'timers/promises'; import { readFile } from 'fs/promises'; @@ -10,25 +10,23 @@ const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests const httpApiHelper = new HttpApiHelper(); When( - /^I call publish on node (\d+) with ([^"]*)/, + /^I call Publish on the node (\d+) with ([^"]*)/, { timeout: 120000 }, async function publish(node, assertionName) { - this.logger.log(`I call publish route on node ${node}`); + this.logger.log(`I call publish route on the node ${node}`); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, ).to.be.equal(true); - const { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey } = - this.state.nodes[node - 1].configuration.modules.blockchain.implementation.hardhat - .config; + const assertion = assertions[assertionName]; const result = await this.state.nodes[node - 1].client - .publish(assertion, { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey }) + .publish(assertion) .catch((error) => { assert.fail(`Error while trying to publish assertion. ${error}`); }); const { operationId } = result.operation; - this.state.lastPublishData = { + this.state.latestPublishData = { nodeId: node - 1, UAL: result.UAL, assertionId: result.assertionId, @@ -40,11 +38,12 @@ When( }; }, ); + When( - /^I call publish on ot-node (\d+) directly with ([^"]*)/, + /^I call Publish directly on the node (\d+) with ([^"]*)/, { timeout: 70000 }, async function publish(node, requestName) { - this.logger.log(`I call publish on ot-node ${node} directly`); + this.logger.log(`I call publish on the node ${node} directly`); expect( !!requests[requestName], `Request body with name: ${requestName} not found!`, @@ -55,63 +54,65 @@ When( requestBody, ); const { operationId } = result.data; - this.state.lastPublishData = { + this.state.latestPublishData = { nodeId: node - 1, operationId, }; }, ); -Given('I wait for last publish to finalize', { timeout: 80000 }, async function publishFinalize() { - this.logger.log('I wait for last publish to finalize'); +When('I wait for latest Publish to finalize', { timeout: 80000 }, async function publishFinalize() { + this.logger.log('I wait for latest publish to finalize'); expect( - !!this.state.lastPublishData, - 'Last publish data is undefined. Publish is not started.', + !!this.state.latestPublishData, + 'Latest publish data is undefined. Publish was not started.', ).to.be.equal(true); - const publishData = this.state.lastPublishData; + const publishData = this.state.latestPublishData; let retryCount = 0; const maxRetryCount = 5; for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { this.logger.log( - `Getting publish result for operation id: ${publishData.operationId} on node: ${publishData.nodeId}`, + `Getting publish result for operation id: ${publishData.operationId} on the node: ${publishData.nodeId}`, ); // eslint-disable-next-line no-await-in-loop const publishResult = await httpApiHelper.getOperationResult( this.state.nodes[publishData.nodeId].nodeRpcUrl, + 'publish', publishData.operationId, ); this.logger.log(`Operation status: ${publishResult.data.status}`); if (['COMPLETED', 'FAILED'].includes(publishResult.data.status)) { - this.state.lastPublishData.result = publishResult; - this.state.lastPublishData.status = publishResult.data.status; - this.state.lastPublishData.errorType = publishResult.data.data?.errorType; + this.state.latestPublishData.result = publishResult; + this.state.latestPublishData.status = publishResult.data.status; + this.state.latestPublishData.errorType = publishResult.data.data?.errorType; break; } if (retryCount === maxRetryCount - 1) { - assert.fail('Unable to get publish result'); + assert.fail('Unable to fetch publish result'); } // eslint-disable-next-line no-await-in-loop await setTimeout(4000); } }); -Given( +When( /I wait for (\d+) seconds and check operation status/, { timeout: 120000 }, async function publishWait(numberOfSeconds) { this.logger.log(`I wait for ${numberOfSeconds} seconds`); expect( - !!this.state.lastPublishData, - 'Last publish data is undefined. Publish is not started.', + !!this.state.latestPublishData, + 'Latest publish data is undefined. Publish is not started.', ).to.be.equal(true); - const publishData = this.state.lastPublishData; + const publishData = this.state.latestPublishData; this.logger.log( - `Getting publish result for operation id: ${publishData.operationId} on node: ${publishData.nodeId}`, + `Getting publish result for operation id: ${publishData.operationId} on the node: ${publishData.nodeId}`, ); await setTimeout(numberOfSeconds * 1000); // eslint-disable-next-line no-await-in-loop - this.state.lastPublishData.result = await httpApiHelper.getOperationResult( + this.state.latestPublishData.result = await httpApiHelper.getOperationResult( this.state.nodes[publishData.nodeId].nodeRpcUrl, + 'publish', publishData.operationId, ); }, diff --git a/test/bdd/steps/api/resolve.mjs b/test/bdd/steps/api/resolve.mjs index eaa7d7cd9a..695715b31c 100644 --- a/test/bdd/steps/api/resolve.mjs +++ b/test/bdd/steps/api/resolve.mjs @@ -1,32 +1,29 @@ -import { When, Given } from '@cucumber/cucumber'; +import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { setTimeout } from 'timers/promises'; import HttpApiHelper from "../../../utilities/http-api-helper.mjs"; -import {readFile} from "fs/promises"; -const requests = JSON.parse(await readFile("test/bdd/steps/api/datasets/requests.json")); const httpApiHelper = new HttpApiHelper() - When( - /^I get operation result from node (\d+) for last published assertion/, + /^I get operation result from node (\d+) for latest published assertion/, { timeout: 120000 }, async function resolveCall(node) { - this.logger.log('I call get result for the last operation'); + this.logger.log('I call get result for the latest operation'); expect( - !!this.state.lastPublishData, - 'Last publish data is undefined. Publish is not finalized.', + !!this.state.latestPublishData, + 'Latest publish data is undefined. Publish is not finalized.', ).to.be.equal(true); try { const result = await this.state.nodes[node - 1].client - .getResult(this.state.lastPublishData.UAL) + .get(this.state.latestPublishData.UAL) .catch((error) => { assert.fail(`Error while trying to resolve assertion. ${error}`); }); const { operationId } = result.operation; - this.state.lastGetData = { + this.state.latestGetData = { nodeId: node - 1, operationId, result, @@ -39,32 +36,33 @@ When( }, ); -Given( - 'I wait for last resolve to finalize', +When( + 'I wait for latest resolve to finalize', { timeout: 120000 }, async function resolveFinalizeCall() { - this.logger.log('I wait for last resolve to finalize'); + this.logger.log('I wait for latest resolve to finalize'); expect( - !!this.state.lastGetData, - 'Last resolve data is undefined. Resolve is not started.', + !!this.state.latestGetData, + 'Latest resolve data is undefined. Resolve is not started.', ).to.be.equal(true); - const resolveData = this.state.lastGetData; + const resolveData = this.state.latestGetData; let retryCount = 0; const maxRetryCount = 5; for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { this.logger.log( - `Getting resolve result for operation id: ${resolveData.operationId} on node: ${resolveData.nodeId}`, + `Getting resolve result for operation id: ${resolveData.operationId} on the node: ${resolveData.nodeId}`, ); // eslint-disable-next-line no-await-in-loop const resolveResult = await httpApiHelper.getOperationResult( this.state.nodes[resolveData.nodeId].nodeRpcUrl, + 'get', resolveData.operationId, ); this.logger.log(`Operation status: ${resolveResult.data.status}`); if (['COMPLETED', 'FAILED'].includes(resolveResult.data.status)) { - this.state.lastGetData.result = resolveResult; - this.state.lastGetData.status = resolveResult.data.status; - this.state.lastGetData.errorType = resolveResult.data.data?.errorType; + this.state.latestGetData.result = resolveResult; + this.state.latestGetData.status = resolveResult.data.status; + this.state.latestGetData.errorType = resolveResult.data.data?.errorType; break; } if (retryCount === maxRetryCount - 1) { @@ -76,17 +74,17 @@ Given( }, ); -Given(/Last resolve returned valid result$/, { timeout: 120000 }, async function resolveCall() { - this.logger.log('Last resolve returned valid result'); +When(/Latest resolve returned valid result$/, { timeout: 120000 }, async function resolveCall() { + this.logger.log('Latest resolve returned valid result'); expect( - !!this.state.lastGetData, - 'Last resolve data is undefined. Resolve is not started.', + !!this.state.latestGetData, + 'Latest resolve data is undefined. Resolve is not started.', ).to.be.equal(true); expect( - !!this.state.lastGetData.result, - 'Last publish data result is undefined. Publish is not finished.', + !!this.state.latestGetData.result, + 'Latest publish data result is undefined. Publish is not finished.', ).to.be.equal(true); - const resolveData = this.state.lastGetData; + const resolveData = this.state.latestGetData; expect( Array.isArray(resolveData.result.data), 'Resolve result data expected to be array', @@ -95,30 +93,7 @@ Given(/Last resolve returned valid result$/, { timeout: 120000 }, async function // expect(resolveData.result.data.length, 'Returned data array length').to.be.equal(1); // const resolvedAssertion = resolveData.result.data[0].assertion.data; - // const publishedAssertion = this.state.lastPublishData.assertion; + // const publishedAssertion = this.state.latestPublishData.assertion; // assert.equal(sortedStringify(publishedAssertion), sortedStringify(resolvedAssertion)); }); -Given( - /^I call get directly to ot-node (\d+) with ([^"]*)/, - { timeout: 30000 }, - async function getFromNode(node, requestName) { - this.logger.log(`I call get on ot-node ${node} directly`); - if (requestName !== 'lastPublishedAssetUAL') { - expect( - !!requests[requestName], - `Request body with name: ${requestName} not found!`, - ).to.be.equal(true); - } - const requestBody = - requestName !== 'lastPublishedAssetUAL' - ? requests[requestName] - : { id: this.state.lastPublishData.UAL }; - const result = await httpApiHelper.get(this.state.nodes[node - 1].nodeRpcUrl, requestBody); - const { operationId } = result.data; - this.state.lastGetData = { - nodeId: node - 1, - operationId, - }; - }, -); diff --git a/test/bdd/steps/api/update.mjs b/test/bdd/steps/api/update.mjs index 7a3f9a84b8..ebf4c46513 100644 --- a/test/bdd/steps/api/update.mjs +++ b/test/bdd/steps/api/update.mjs @@ -1,4 +1,4 @@ -import { When, Given } from '@cucumber/cucumber'; +import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { setTimeout } from 'timers/promises'; import { readFile } from 'fs/promises'; @@ -10,26 +10,24 @@ const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests const httpApiHelper = new HttpApiHelper(); When( - /^I call update on node (\d+) for last publish UAL with ([^"]*)/, + /^I call Update on the node (\d+) for the latest published UAL with ([^"]*)/, { timeout: 120000 }, async function update(node, assertionName) { - this.logger.log(`I call update route on node ${node}`); + this.logger.log(`I call update route on the node ${node}`); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, ).to.be.equal(true); - const { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey } = - this.state.nodes[node - 1].configuration.modules.blockchain.implementation.hardhat - .config; + const assertion = assertions[assertionName]; - const { UAL } = this.state.lastPublishData; + const { UAL } = this.state.latestPublishData; const result = await this.state.nodes[node - 1].client - .update(UAL, assertion, { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey }) + .update(UAL, assertion) .catch((error) => { assert.fail(`Error while trying to update assertion. ${error}`); }); const { operationId } = result.operation; - this.state.lastUpdateData = { + this.state.latestUpdateData = { nodeId: node - 1, UAL, assertionId: result.assertionId, @@ -41,11 +39,12 @@ When( }; }, ); + When( - /^I call update on ot-node (\d+) directly with ([^"]*)/, + /^I call Update directly on the node (\d+) with ([^"]*)/, { timeout: 70000 }, async function publish(node, requestName) { - this.logger.log(`I call update on ot-node ${node} directly`); + this.logger.log(`I call update on the node ${node} directly`); expect( !!requests[requestName], `Request body with name: ${requestName} not found!`, @@ -56,40 +55,41 @@ When( requestBody, ); const { operationId } = result.data; - this.state.lastUpdateData = { + this.state.latestUpdateData = { nodeId: node - 1, operationId, }; }, ); -Given('I wait for last update to finalize', { timeout: 80000 }, async function publishFinalize() { - this.logger.log('I wait for last update to finalize'); +When('I wait for latest Update to finalize', { timeout: 80000 }, async function publishFinalize() { + this.logger.log('I wait for latest update to finalize'); expect( - !!this.state.lastUpdateData, - 'Last update data is undefined. Update is not started.', + !!this.state.latestUpdateData, + 'Latest update data is undefined. Update was not started.', ).to.be.equal(true); - const updateData = this.state.lastUpdateData; + const updateData = this.state.latestUpdateData; let retryCount = 0; const maxRetryCount = 5; for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { this.logger.log( - `Getting Update result for operation id: ${updateData.operationId} on node: ${updateData.nodeId}`, + `Getting Update result for operation id: ${updateData.operationId} on the node: ${updateData.nodeId}`, ); // eslint-disable-next-line no-await-in-loop const updateResult = await httpApiHelper.getOperationResult( this.state.nodes[updateData.nodeId].nodeRpcUrl, + 'update', updateData.operationId, ); this.logger.log(`Operation status: ${updateResult.data.status}`); if (['COMPLETED', 'FAILED'].includes(updateResult.data.status)) { - this.state.lastUpdateData.result = updateResult; - this.state.lastUpdateData.status = updateResult.data.status; - this.state.lastUpdateData.errorType = updateResult.data.data?.errorType; + this.state.latestUpdateData.result = updateResult; + this.state.latestUpdateData.status = updateResult.data.status; + this.state.latestUpdateData.errorType = updateResult.data.data?.errorType; break; } if (retryCount === maxRetryCount - 1) { - assert.fail('Unable to get update result'); + assert.fail('Unable to fetch update result'); } // eslint-disable-next-line no-await-in-loop await setTimeout(4000); diff --git a/test/bdd/steps/common.mjs b/test/bdd/steps/common.mjs index c2a3c5138e..5a09f88575 100644 --- a/test/bdd/steps/common.mjs +++ b/test/bdd/steps/common.mjs @@ -1,10 +1,11 @@ -import { Given } from '@cucumber/cucumber'; +import { Given, Then } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import fs from 'fs'; import { setTimeout as sleep } from 'timers/promises'; import DkgClientHelper from '../../utilities/dkg-client-helper.mjs'; import StepsUtils from '../../utilities/steps-utils.mjs'; +import FileService from "../../../src/service/file-service.js"; const stepsUtils = new StepsUtils(); @@ -53,15 +54,21 @@ Given( const client = new DkgClientHelper({ endpoint: 'http://localhost', port: rpcPort, - useSSL: false, - timeout: 25, - loglevel: 'trace', + blockchain: { + name: 'hardhat', + publicKey: wallet.address, + privateKey: wallet.privateKey, + }, + maxNumberOfRetries: 5, + frequency: 2, + contentType: 'all', }); this.state.nodes[nodeIndex] = { client, forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, + fileService: new FileService({config: nodeConfiguration, logger: this.logger}), }; } nodesStarted += 1; @@ -125,6 +132,7 @@ Given( forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, + fileService: new FileService({config: nodeConfiguration, logger: this.logger}), }); } done(); @@ -162,9 +170,8 @@ Given( Object.prototype.hasOwnProperty.call(nodeConfiguration, propertyNameSplit[0]), `Property ${propertyName} doesn't exist`, ).to.be.equal(true); - const propertyNameSplitLen = propertyNameSplit.length; let propName = nodeConfiguration; - for (let i = 0; i < propertyNameSplitLen - 1; i += 1) { + for (let i = 0; i < propertyNameSplit.length - 1; i += 1) { propName = propName[propertyNameSplit[i]]; } if (propName[propertyNameSplit.slice(-1)] !== undefined) { @@ -189,41 +196,41 @@ Given( const client = new DkgClientHelper({ endpoint: 'http://localhost', port: rpcPort, - useSSL: false, - timeout: 25, - loglevel: 'trace', + blockchain: { + name: 'hardhat', + publicKey: wallet.address, + privateKey: wallet.privateKey, + }, + maxNumberOfRetries: 5, + frequency: 2, + contentType: 'all', }); this.state.nodes[nodeIndex] = { client, - clientConfig: { - endpoint: 'http://localhost', - port: rpcPort, - useSSL: false, - timeout: 25, - loglevel: 'trace', - }, forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, + fileService: new FileService({config: nodeConfiguration, logger: this.logger}), }; } done(); }); }, ); -Given( - /Last (Get|Publish|Update) operation finished with status: ([COMPLETED|FAILED|PublishValidateAssertionError|PublishStartError|GetAssertionIdError|GetNetworkError|GetLocalError|PublishRouteError]+)$/, + +Then( + /Latest (Get|Publish|Update) operation finished with status: ([COMPLETED|FAILED|PublishValidateAssertionError|PublishStartError|GetAssertionIdError|GetNetworkError|GetLocalError|PublishRouteError]+)$/, { timeout: 120000 }, - async function lastResolveFinishedCall(operationName, status) { - this.logger.log(`Last ${operationName} operation finished with status: ${status}`); - const operationData = `last${operationName}Data`; + async function latestResolveFinishedCall(operationName, status) { + this.logger.log(`Latest ${operationName} operation finished with status: ${status}`); + const operationData = `latest${operationName}Data`; expect( !!this.state[operationData], - `Last ${operationName} result is undefined. ${operationData} result not started.`, + `Latest ${operationName} result is undefined. ${operationData} result not started.`, ).to.be.equal(true); expect( !!this.state[operationData].result, - `Last ${operationName} result data result is undefined. ${operationData} result is not finished.`, + `Latest ${operationName} result data result is undefined. ${operationData} result is not finished.`, ).to.be.equal(true); expect( @@ -247,3 +254,8 @@ Given(/^I set R0 to be (\d+)$/, { timeout: 100000 }, async function waitFor(r0) this.logger.log(`I set R0 to be ${r0}`); await this.state.localBlockchain.setR0(r0); }); + +Given(/^I set finalizationCommitsNumber to be (\d+)$/, { timeout: 100000 }, async function waitFor(finalizationCommitsNumber) { + this.logger.log(`I set finalizationCommitsNumber to be ${finalizationCommitsNumber}`); + await this.state.localBlockchain.setFinalizationCommitsNumber(finalizationCommitsNumber); +}); diff --git a/test/bdd/steps/hooks.mjs b/test/bdd/steps/hooks.mjs index 27c9fc9856..9fbb4391c2 100644 --- a/test/bdd/steps/hooks.mjs +++ b/test/bdd/steps/hooks.mjs @@ -2,9 +2,9 @@ import 'dotenv/config'; import { Before, BeforeAll, After, AfterAll } from '@cucumber/cucumber'; import slugify from 'slugify'; import fs from 'fs'; +import mysql from "mysql2"; import { NODE_ENVIRONMENTS } from '../../../src/constants/constants.js'; import TripleStoreModuleManager from "../../../src/modules/triple-store/triple-store-module-manager.js"; -import mysql from "mysql2"; process.env.NODE_ENV = NODE_ENVIRONMENTS.TEST; @@ -29,22 +29,28 @@ Before(function beforeMethod(testCase, done) { After(function afterMethod(testCase, done) { const tripleStoreConfiguration = []; const databaseNames = []; + const promises = []; for (const key in this.state.nodes) { this.state.nodes[key].forkedNode.kill(); tripleStoreConfiguration.push({modules: {tripleStore: this.state.nodes[key].configuration.modules.tripleStore}}); databaseNames.push(this.state.nodes[key].configuration.operationalDatabase.databaseName); + const dataFolderPath = this.state.nodes[key].fileService.getDataFolderPath(); + promises.push(this.state.nodes[key].fileService.removeFolder(dataFolderPath)); } this.state.bootstraps.forEach((node) => { node.forkedNode.kill(); tripleStoreConfiguration.push({modules: {tripleStore: node.configuration.modules.tripleStore}}); databaseNames.push(node.configuration.operationalDatabase.databaseName); + const dataFolderPath = node.fileService.getDataFolderPath(); + promises.push(node.fileService.removeFolder(dataFolderPath)); }); if (this.state.localBlockchain) { + this.logger.info('Stopping local blockchain!'); this.state.localBlockchain.stop(); + this.state.localBlockchain = null; } this.logger.log('After test hook, cleaning repositories'); - const promises = []; const con = mysql.createConnection({ host: 'localhost', user: 'root', @@ -58,18 +64,17 @@ After(function afterMethod(testCase, done) { tripleStoreConfiguration.forEach((config) => { promises.push(async () => { const tripleStoreModuleManager = new TripleStoreModuleManager({config, logger: this.logger}); - await tripleStoreModuleManager.initialize() + await tripleStoreModuleManager.initialize(); for (const implementationName of tripleStoreModuleManager.getImplementationNames()) { - const {config} = tripleStoreModuleManager.getImplementation(implementationName); - Object.keys(config.repositories).map(async (repository) => { - console.log('Removing triple store configuration:', JSON.stringify(config, null, 4)); + const {tripleStoreConfig} = tripleStoreModuleManager.getImplementation(implementationName); + Object.keys(tripleStoreConfig.repositories).map(async (repository) => { + this.logger.log('Removing triple store configuration:', JSON.stringify(tripleStoreConfig, null, 4)); await tripleStoreModuleManager.deleteRepository(implementationName, repository); } ) } }) }) - Promise.all(promises) .then(() => { con.end(); diff --git a/test/bdd/steps/lib/local-blockchain.mjs b/test/bdd/steps/lib/local-blockchain.mjs index aeb7993061..65515e49eb 100644 --- a/test/bdd/steps/lib/local-blockchain.mjs +++ b/test/bdd/steps/lib/local-blockchain.mjs @@ -115,18 +115,27 @@ class LocalBlockchain { } } - async setR1(R1) { - console.log(`Setting R1 in parameters storage to: ${R1}`); - const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR1', [R1]); + async setR0(r0) { + console.log(`Setting R0 in parameters storage to: ${r0}`); + const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR0', [r0]); const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); await this.HubControllerContract.forwardCall(parametersStorageAddress, encodedData); } - async setR0(R0) { - console.log(`Setting R0 in parameters storage to: ${R0}`); - const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR0', [R0]); + async setR1(r1) { + console.log(`Setting R1 in parameters storage to: ${r1}`); + const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR1', [r1]); + const parametersStorageAddress = await this.hubContract.getContractAddress( + 'ParametersStorage', + ); + await this.HubControllerContract.forwardCall(parametersStorageAddress, encodedData); + } + + async setFinalizationCommitsNumber(commitsNumber) { + console.log(`Setting finalizationCommitsNumber in parameters storage to: ${commitsNumber}`); + const encodedData = this.ParametersStorageInterface.encodeFunctionData('setFinalizationCommitsNumber', [commitsNumber]); const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); diff --git a/test/bdd/steps/lib/state.mjs b/test/bdd/steps/lib/state.mjs index 2fcc82bc04..727635eeb3 100644 --- a/test/bdd/steps/lib/state.mjs +++ b/test/bdd/steps/lib/state.mjs @@ -12,31 +12,38 @@ const state = { 0: { client: {}, fork: {}, + fileService: {}, + configuration: {}, + nodeRpcUrl: '' }, 1: { client: {}, fork: {}, + fileService: {}, + configuration: {}, + nodeRpcUrl: '' }, }, bootstraps: [], - lastPublishData: { + latestPublishData: { nodeId: 1, operationId: '', keywords: ['', ''], assertion: {}, result: {}, }, - lastGetData: { + latestGetData: { nodeId: 1, operationId: '', assertionIds: ['', ''], result: {}, }, - lastUpdateData: { + latestUpdateData: { nodeId: 1, operationId: '', assertionIds: ['', ''], result: {}, }, + latestError: {}, scenarionLogDir: '', }; diff --git a/test/unit/service/publish-service.test.js b/test/unit/service/publish-service.test.js index 07a4d60523..fad7dc6adf 100644 --- a/test/unit/service/publish-service.test.js +++ b/test/unit/service/publish-service.test.js @@ -59,7 +59,7 @@ describe('Publish service test', async () => { const returnedResponses = publishService.repositoryModuleManager.getAllResponseStatuses(); expect(cacheOperationIdDataSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', {})).to.be - .true; + .false; expect(returnedResponses.length).to.be.equal(2); diff --git a/test/unit/service/update-service.test.js b/test/unit/service/update-service.test.js index 4e524cde3d..46409a8566 100644 --- a/test/unit/service/update-service.test.js +++ b/test/unit/service/update-service.test.js @@ -62,7 +62,7 @@ describe('Update service test', async () => { expect(returnedResponses.length).to.be.equal(2); expect(cacheOperationIdDataSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', {})).to.be - .true; + .false; expect( returnedResponses[returnedResponses.length - 1].status === diff --git a/test/utilities/dkg-client-helper.mjs b/test/utilities/dkg-client-helper.mjs index c9c7e172fe..bc18895fcd 100644 --- a/test/utilities/dkg-client-helper.mjs +++ b/test/utilities/dkg-client-helper.mjs @@ -10,52 +10,35 @@ class DkgClientHelper { return this.client.node.info(); } - async publish(data, wallet) { + async publish(data) { const options = { visibility: 'public', epochsNum: 5, - maxNumberOfRetries: 5, hashFunctionId: CONTENT_ASSET_HASH_FUNCTION_ID, - blockchain: { - name: 'hardhat', - publicKey: wallet.evmOperationalWalletPublicKey, - privateKey: wallet.evmOperationalWalletPrivateKey, - }, }; + return this.client.asset.create(data, options); } - async update(ual, assertion, wallet) { + async update(ual, assertion) { const options = { - maxNumberOfRetries: 5, - blockchain: { - name: 'hardhat', - publicKey: wallet.evmOperationalWalletPublicKey, - privateKey: wallet.evmOperationalWalletPrivateKey, - }, + hashFunctionId: CONTENT_ASSET_HASH_FUNCTION_ID, }; + return this.client.asset.update(ual, assertion, options); } - async get(ids) { - return this.client._getRequest({ - ids, - }); - } + async get(ual, state) { + const options = { + state, + validate: true, + }; - async query(query) { - return this.client._queryRequest({ - query, - }); + return this.client.asset.get(ual, options); } - async getResult(UAL) { - const getOptions = { - validate: true, - commitOffset: 0, - maxNumberOfRetries: 5, - }; - return this.client.asset.get(UAL, getOptions).catch(() => {}); + async query(query) { + return this.client.query(query); } } diff --git a/test/utilities/http-api-helper.mjs b/test/utilities/http-api-helper.mjs index aabcc71929..bac9c33d99 100644 --- a/test/utilities/http-api-helper.mjs +++ b/test/utilities/http-api-helper.mjs @@ -2,69 +2,36 @@ import axios from 'axios'; class HttpApiHelper { async info(nodeRpcUrl) { - try { - const response = await axios({ - method: 'get', - url: `${nodeRpcUrl}/info`, - }); - - return response; - } catch (e) { - throw Error(`Unable to get info: ${e.message}`); - } + return this._sendRequest('get', `${nodeRpcUrl}/info`); } - async get(nodeRpcUrl, ual) { - // Not sure if header is needed - try { - const response = await axios({ - method: 'post', - url: `${nodeRpcUrl}/get`, - data: ual, - headers: { - 'Content-Type': 'application/json', - }, - }); - - return response; - } catch (e) { - throw Error(`Unable to GET: ${e.message}`); - } + async get(nodeRpcUrl, requestBody) { + return this._sendRequest('post', `${nodeRpcUrl}/get`, requestBody); } - async getOperationResult(nodeRpcUrl, operationId) { - try { - const response = await axios({ - method: 'get', - url: `${nodeRpcUrl}/publish/${operationId}`, - }); - - return response; - } catch (e) { - throw Error(`Unable to PUBLISH: ${e.message}`); - } + async getOperationResult(nodeRpcUrl, operationName, operationId) { + return this._sendRequest('get', `${nodeRpcUrl}/${operationName}/${operationId}`); } async publish(nodeRpcUrl, requestBody) { - try { - const response = await axios({ - method: 'post', - url: `${nodeRpcUrl}/publish`, - data: requestBody, - }); - - return response; - } catch (e) { - throw Error(`Unable to publish: ${e.message}`); - } + return this._sendRequest('post', `${nodeRpcUrl}/publish`, requestBody); } + async update(nodeRpcUrl, requestBody) { + return this._sendRequest('post', `${nodeRpcUrl}/update`, requestBody); + } + + async _sendRequest(method, url, data) { return axios({ - method: 'post', - url: `${nodeRpcUrl}/update`, - data: requestBody, - }).catch((e) => { - throw Error(`Unable to update: ${e.message}`); + method, + url, + ...data && { data }, + }).catch((error) => { + const errorWithStatus = new Error(error.message); + if (error.response) { + errorWithStatus.statusCode = error.response.status; + } + throw errorWithStatus; }); } } diff --git a/test/utilities/steps-utils.mjs b/test/utilities/steps-utils.mjs index cf79f4e1a7..e17a5eb646 100644 --- a/test/utilities/steps-utils.mjs +++ b/test/utilities/steps-utils.mjs @@ -124,10 +124,6 @@ class StepsUtils { graphDatabase: { name: nodeName, }, - minimumAckResponses: { - publish: 2, - get: 1, - }, }; } }