diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 64356fb2f4..f7dee6cf44 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -2,8 +2,7 @@ name: checks on: pull_request: - branches: - - v6/develop + types: [opened, reopened, synchronize] env: REPOSITORY_PASSWORD: password diff --git a/config/config.json b/config/config.json index 5cbff178d5..8f3679f4c2 100644 --- a/config/config.json +++ b/config/config.json @@ -168,6 +168,7 @@ } } }, + "maximumAssertionSizeInKb": 2500, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logLevel": "info", @@ -302,6 +303,7 @@ } } }, + "maximumAssertionSizeInKb": 2500, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logLevel": "trace", @@ -449,6 +451,7 @@ } } }, + "maximumAssertionSizeInKb": 2500, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logLevel": "trace", @@ -597,6 +600,7 @@ } } }, + "maximumAssertionSizeInKb": 2500, "commandExecutorVerboseLoggingEnabled": false, "appDataPath": "data", "logLevel": "trace", diff --git a/ot-node.js b/ot-node.js index 2660f544ad..5e89a6b99f 100644 --- a/ot-node.js +++ b/ot-node.js @@ -69,6 +69,7 @@ class OTNode { await this.initializeCommandExecutor(); await this.initializeRouters(); + await this.startNetworkModule(); this.logger.info('Node is up and running!'); } @@ -254,6 +255,11 @@ class OTNode { } } + async startNetworkModule() { + const networkModuleManager = this.container.resolve('networkModuleManager'); + await networkModuleManager.start(); + } + async executePrivateAssetsMetadataMigration() { if ( process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || diff --git a/package-lock.json b/package-lock.json index 1e48c518dc..85d120a487 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.12", + "version": "6.0.13", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.12", + "version": "6.0.13", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", @@ -25,7 +25,7 @@ "axios": "^0.27.2", "cors": "^2.8.5", "deep-extend": "^0.6.0", - "dkg-evm-module": "^4.0.5", + "dkg-evm-module": "^4.0.7", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", @@ -7612,9 +7612,9 @@ } }, "node_modules/dkg-evm-module": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/dkg-evm-module/-/dkg-evm-module-4.0.5.tgz", - "integrity": "sha512-hm6MbTRnuZWrgKOIgGm1iP6JgUNSg3Q9hMsEYtA6Qd9M3ns7N20cHwtp7XMiZx/Mypx9NAsY6VaxaN3PfndSog==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/dkg-evm-module/-/dkg-evm-module-4.0.7.tgz", + "integrity": "sha512-ZjrGEvdLpirmIHkuA3L/bEo7IkbJwHi6F33U7GMYEwvr/9hsgNnV3O5iiegb5UpISu8SgN0eKOS9hlr7ty5WzQ==", "dependencies": { "@openzeppelin/contracts": "^4.7.3", "@polkadot/api": "^10.1.4", @@ -19096,9 +19096,9 @@ } }, "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", + "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", "engines": { "node": ">=0.10.0" } @@ -25623,9 +25623,9 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" }, "dkg-evm-module": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/dkg-evm-module/-/dkg-evm-module-4.0.5.tgz", - "integrity": "sha512-hm6MbTRnuZWrgKOIgGm1iP6JgUNSg3Q9hMsEYtA6Qd9M3ns7N20cHwtp7XMiZx/Mypx9NAsY6VaxaN3PfndSog==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/dkg-evm-module/-/dkg-evm-module-4.0.7.tgz", + "integrity": "sha512-ZjrGEvdLpirmIHkuA3L/bEo7IkbJwHi6F33U7GMYEwvr/9hsgNnV3O5iiegb5UpISu8SgN0eKOS9hlr7ty5WzQ==", "requires": { "@openzeppelin/contracts": "^4.7.3", "@polkadot/api": "^10.1.4", @@ -34634,9 +34634,9 @@ } }, "word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", + "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==" }, "wrap-ansi": { "version": "7.0.0", diff --git a/package.json b/package.json index c43d91afb6..5ce51dfe14 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.12", + "version": "6.0.13", "description": "OTNode V6", "main": "index.js", "type": "module", @@ -12,6 +12,7 @@ "lint-staged": "lint-staged", "create-account-mapping-signature": "node tools/ot-parachain-account-mapping/create-account-mapping-signature.js ", "start:local_blockchain": "npm explore dkg-evm-module -- npm run dev", + "kill:local_blockchain": "npx kill-port 8545", "test:bdd": "cucumber-js --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ --import test/bdd/steps/ --exit", "test:unit": "nyc --all mocha --exit $(find test/unit -name '*.js')", "test:modules": "nyc --all mocha --exit $(find test/modules -name '*.js')", @@ -74,7 +75,7 @@ "axios": "^0.27.2", "cors": "^2.8.5", "deep-extend": "^0.6.0", - "dkg-evm-module": "^4.0.5", + "dkg-evm-module": "^4.0.7", "dotenv": "^16.0.1", "ethers": "^5.7.2", "express": "^4.18.1", diff --git a/src/commands/local-store/local-store-command.js b/src/commands/local-store/local-store-command.js index 8719f8d322..5bf84abc00 100644 --- a/src/commands/local-store/local-store-command.js +++ b/src/commands/local-store/local-store-command.js @@ -93,8 +93,11 @@ class LocalStoreCommand extends Command { await this.commandExecutor.add({ name: 'deletePendingStateCommand', sequence: [], - delay: updateCommitWindowDuration * 1000, - data: { ...command.data, repository: PENDING_STORAGE_REPOSITORIES.PRIVATE }, + delay: (updateCommitWindowDuration + 60) * 1000, + data: { + ...command.data, + assertionId: cachedData.public.assertionId, + }, transactional: false, }); } diff --git a/src/commands/protocols/common/handle-protocol-message-command.js b/src/commands/protocols/common/handle-protocol-message-command.js index 43b74e4e8a..79400fda6e 100644 --- a/src/commands/protocols/common/handle-protocol-message-command.js +++ b/src/commands/protocols/common/handle-protocol-message-command.js @@ -1,5 +1,5 @@ import Command from '../../command.js'; -import { NETWORK_MESSAGE_TYPES } from '../../../constants/constants.js'; +import { BYTES_IN_KILOBYTE, NETWORK_MESSAGE_TYPES } from '../../../constants/constants.js'; class HandleProtocolMessageCommand extends Command { constructor(ctx) { @@ -127,6 +127,18 @@ class HandleProtocolMessageCommand extends Command { this.blockchainModuleManager.getR0(blockchain), getAsk(), ]); + const blockchainAssertionSizeInKb = blockchainAssertionSize / BYTES_IN_KILOBYTE; + if (blockchainAssertionSizeInKb > this.config.maximumAssertionSizeInKb) { + this.logger.warn( + `The size of the received assertion exceeds the maximum limit allowed.. Maximum allowed assertion size in kb: ${this.config.maximumAssertionSizeInKb}, assertion size read from blockchain in kb: ${blockchainAssertionSizeInKb}`, + ); + return { + errorMessage: + 'The size of the received assertion exceeds the maximum limit allowed.', + agreementId, + agreementData, + }; + } const now = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); @@ -143,8 +155,7 @@ class HandleProtocolMessageCommand extends Command { .mul(epochsLeft) .mul(blockchainAssertionSize); - const serviceAgreementBid = this.blockchainModuleManager - .toBigNumber(blockchain, agreementData.tokenAmount) + const serviceAgreementBid = agreementData.tokenAmount .add(agreementData.updateTokenAmount) .mul(1024) .div(divisor) diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js index e8bcd7aee4..8dbef0a104 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js @@ -42,22 +42,24 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { blockchain, contract, tokenId, + assertionId, ); } - for (const repository of [ - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, - ]) { - if (assertionExists) { - break; + if (!assertionExists) { + for (const repository of [ + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, + ]) { + // eslint-disable-next-line no-await-in-loop + assertionExists = await this.tripleStoreService.assertionExists( + repository, + assertionId, + ); + if (assertionExists) { + break; + } } - - // eslint-disable-next-line no-await-in-loop - assertionExists = await this.tripleStoreService.assertionExists( - repository, - assertionId, - ); } await this.operationIdService.updateOperationIdStatus( diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js index 55adc3989e..576dbe8a5a 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js @@ -26,6 +26,7 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { OPERATION_ID_STATUS.GET.GET_REMOTE_START, ); + let nquads; if ( state !== GET_STATES.FINALIZED && blockchain != null && @@ -37,26 +38,24 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { blockchain, contract, tokenId, + assertionId, operationId, ); if (cachedAssertion?.public?.assertion?.length) { - return { - messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, - messageData: { nquads: cachedAssertion.public.assertion }, - }; + nquads = cachedAssertion.public.assertion; } } - let nquads; - for (const repository of [ - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, - ]) { - // eslint-disable-next-line no-await-in-loop - nquads = await this.tripleStoreService.getAssertion(repository, assertionId); - - if (nquads.length) { - break; + if (!nquads?.length) { + for (const repository of [ + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, + ]) { + // eslint-disable-next-line no-await-in-loop + nquads = await this.tripleStoreService.getAssertion(repository, assertionId); + if (nquads.length) { + break; + } } } diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js index b4af9d93d3..cf40c8d681 100644 --- a/src/commands/protocols/get/sender/get-assertion-id-command.js +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -75,10 +75,7 @@ class GetAssertionIdCommand extends Command { } } - return this.continueSequence( - { ...command.data, state: assertionId, assertionId }, - command.sequence, - ); + return this.continueSequence({ ...command.data, state, assertionId }, command.sequence); } async handleError(operationId, errorMessage, errorType) { diff --git a/src/commands/protocols/get/sender/local-get-command.js b/src/commands/protocols/get/sender/local-get-command.js index f9d09cf0f8..8d64f8186e 100644 --- a/src/commands/protocols/get/sender/local-get-command.js +++ b/src/commands/protocols/get/sender/local-get-command.js @@ -2,6 +2,7 @@ import Command from '../../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, + GET_STATES, TRIPLE_STORE_REPOSITORIES, PENDING_STORAGE_REPOSITORIES, } from '../../../../constants/constants.js'; @@ -24,42 +25,50 @@ class LocalGetCommand extends Command { * @param command */ async execute(command) { - const { operationId, blockchain, contract, tokenId, state } = command.data; + const { operationId, blockchain, contract, tokenId, assertionId, state } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.GET.GET_LOCAL_START, ); const response = {}; - for (const repository of [ - PENDING_STORAGE_REPOSITORIES.PRIVATE, - PENDING_STORAGE_REPOSITORIES.PUBLIC, - ]) { - // eslint-disable-next-line no-await-in-loop - const stateIsPending = await this.pendingStorageService.assetHasPendingState( - repository, - blockchain, - contract, - tokenId, - state, - ); - - if (stateIsPending) { + if ( + state !== GET_STATES.FINALIZED && + blockchain != null && + contract != null && + tokenId != null + ) { + for (const repository of [ + PENDING_STORAGE_REPOSITORIES.PRIVATE, + PENDING_STORAGE_REPOSITORIES.PUBLIC, + ]) { // eslint-disable-next-line no-await-in-loop - const cachedAssertion = await this.pendingStorageService.getCachedAssertion( + const stateIsPending = await this.pendingStorageService.assetHasPendingState( repository, blockchain, contract, tokenId, - operationId, + assertionId, ); - if (cachedAssertion?.public?.assertion?.length) { - response.assertion = cachedAssertion.public.assertion; - if (cachedAssertion?.private?.assertion?.length) { - response.privateAssertion = cachedAssertion.private.assertion; + if (stateIsPending) { + // eslint-disable-next-line no-await-in-loop + const cachedAssertion = await this.pendingStorageService.getCachedAssertion( + repository, + blockchain, + contract, + tokenId, + assertionId, + operationId, + ); + + if (cachedAssertion?.public?.assertion?.length) { + response.assertion = cachedAssertion.public.assertion; + if (cachedAssertion?.private?.assertion?.length) { + response.privateAssertion = cachedAssertion.private.assertion; + } + break; } - break; } } } @@ -72,7 +81,10 @@ class LocalGetCommand extends Command { TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, ]) { // eslint-disable-next-line no-await-in-loop - response.assertion = await this.tripleStoreService.getAssertion(repository, state); + response.assertion = await this.tripleStoreService.getAssertion( + repository, + assertionId, + ); if (response?.assertion?.length) break; } } diff --git a/src/commands/protocols/publish/sender/publish-schedule-messages-command.js b/src/commands/protocols/publish/sender/publish-schedule-messages-command.js index 8e048be7a5..f5314aca85 100644 --- a/src/commands/protocols/publish/sender/publish-schedule-messages-command.js +++ b/src/commands/protocols/publish/sender/publish-schedule-messages-command.js @@ -1,4 +1,5 @@ import ProtocolScheduleMessagesCommand from '../../common/protocol-schedule-messages-command.js'; +import Command from '../../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE } from '../../../../constants/constants.js'; class PublishScheduleMessagesCommand extends ProtocolScheduleMessagesCommand { @@ -10,6 +11,105 @@ class PublishScheduleMessagesCommand extends ProtocolScheduleMessagesCommand { this.errorType = ERROR_TYPE.PUBLISH.PUBLISH_START_ERROR; } + async execute(command) { + const { + operationId, + keyword, + leftoverNodes, + numberOfFoundNodes, + blockchain, + minAckResponses, + hashFunctionId, + assertionId, + tokenId, + contract, + } = command.data; + let isValid = true; + // perform check only first time not for every batch + if (leftoverNodes === numberOfFoundNodes) { + isValid = await this.validateBidsForNeighbourhood( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + assertionId, + leftoverNodes, + minAckResponses, + operationId, + ); + } + if (isValid) { + return super.execute(command); + } + return Command.empty(); + } + + async validateBidsForNeighbourhood( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + assertionId, + nodes, + minAckResponses, + operationId, + ) { + const agreementId = await this.serviceAgreementService.generateId( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + ); + + const agreementData = await this.blockchainModuleManager.getAgreementData( + blockchain, + agreementId, + ); + + const r0 = await this.blockchainModuleManager.getR0(blockchain); + + const blockchainAssertionSize = await this.blockchainModuleManager.getAssertionSize( + blockchain, + assertionId, + ); + + const divisor = this.blockchainModuleManager + .toBigNumber(blockchain, r0) + .mul(Number(agreementData.epochsNumber)) + .mul(blockchainAssertionSize); + + const serviceAgreementBid = this.blockchainModuleManager + .toBigNumber(blockchain, agreementData.tokenAmount) + .add(agreementData.updateTokenAmount) + .mul(1024) + .div(divisor) + .add(1); // add 1 wei because of the precision loss + + let validBids = 0; + + nodes.forEach((node) => { + const askNumber = this.blockchainModuleManager.convertToWei(blockchain, node.ask); + + const ask = this.blockchainModuleManager.toBigNumber(blockchain, askNumber); + + if (ask.lte(serviceAgreementBid)) { + validBids += 1; + } + }); + if (validBids < minAckResponses) { + await this.operationService.markOperationAsFailed( + operationId, + 'Unable to start publish, not enough nodes in neighbourhood satisfy the bid.', + ERROR_TYPE.PUBLISH.PUBLISH_START_ERROR, + ); + return false; + } + return true; + } + /** * Builds default publishScheduleMessagesCommand * @param map diff --git a/src/commands/protocols/update/receiver/delete-pending-state-command.js b/src/commands/protocols/update/receiver/delete-pending-state-command.js index a8b824e674..defa18d8d2 100644 --- a/src/commands/protocols/update/receiver/delete-pending-state-command.js +++ b/src/commands/protocols/update/receiver/delete-pending-state-command.js @@ -1,25 +1,64 @@ import Command from '../../../command.js'; -import { ERROR_TYPE } from '../../../../constants/constants.js'; +import { ERROR_TYPE, PENDING_STORAGE_REPOSITORIES } from '../../../../constants/constants.js'; class DeletePendingStateCommand extends Command { constructor(ctx) { super(ctx); + this.blockchainModuleManager = ctx.blockchainModuleManager; this.pendingStorageService = ctx.pendingStorageService; this.errorType = ERROR_TYPE.UPDATE.UPDATE_DELETE_PENDING_STATE_ERROR; } async execute(command) { - const { blockchain, contract, tokenId, operationId, repository } = command.data; + const { blockchain, contract, tokenId, assertionId, operationId } = command.data; - await this.pendingStorageService.removeCachedAssertion( - repository, + this.logger.trace( + `Started ${command.name} for blockchain: ${blockchain} contract: ${contract}, ` + + `token id: ${tokenId}, assertion id: ${assertionId}`, + ); + + const assetStates = await this.blockchainModuleManager.getAssertionIds( blockchain, contract, tokenId, - operationId, ); + if (assetStates.includes(assertionId)) { + this.logger.trace( + `Not clearing the pending storage as state was finalized and clearing is triggered by StateFinalized event.`, + ); + return Command.empty(); + } + + for (const repository of [ + PENDING_STORAGE_REPOSITORIES.PUBLIC, + PENDING_STORAGE_REPOSITORIES.PRIVATE, + ]) { + // eslint-disable-next-line no-await-in-loop + const pendingStateExists = await this.pendingStorageService.assetHasPendingState( + repository, + blockchain, + contract, + tokenId, + assertionId, + ); + + if (!pendingStateExists) { + continue; + } + + // eslint-disable-next-line no-await-in-loop + await this.pendingStorageService.removeCachedAssertion( + repository, + blockchain, + contract, + tokenId, + assertionId, + operationId, + ); + } + return Command.empty(); } diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 83eff51b71..0429b3c1f3 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -100,8 +100,11 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { this.commandExecutor.add({ name: 'deletePendingStateCommand', sequence: [], - delay: updateCommitWindowDuration * 1000, - data: { ...commandData, repository: PENDING_STORAGE_REPOSITORIES.PUBLIC }, + delay: (updateCommitWindowDuration + 60) * 1000, + data: { + ...commandData, + assertionId: cachedData.assertionId, + }, transactional: false, }), ); diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 6a30def3b1..e003f881d8 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -371,6 +371,9 @@ class Web3Service { TRANSACTION_CONFIRMATIONS, TRANSACTION_POLLING_TIMEOUT_MILLIS, ); + if (result?.status === 0) { + throw Error(); + } } catch (error) { this.logger.warn( `Failed executing smart contract function ${functionName}. Error: ${error.message}`, diff --git a/src/modules/network/implementation/libp2p-service.js b/src/modules/network/implementation/libp2p-service.js index 1b99a1c873..da50372d0a 100644 --- a/src/modules/network/implementation/libp2p-service.js +++ b/src/modules/network/implementation/libp2p-service.js @@ -101,11 +101,14 @@ class Libp2pService { */ this.sessions = {}; this.node = await libp2p.create(initializationObject); - await this.node.start(); - const port = parseInt(this.node.multiaddrs.toString().split('/')[4], 10); const peerId = this.node.peerId.toB58String(); this.config.id = peerId; - this.logger.info(`Network ID is ${peerId}, connection port is ${port}`); + } + + async start() { + await this.node.start(); + const port = parseInt(this.node.multiaddrs.toString().split('/')[4], 10); + this.logger.info(`Network ID is ${this.config.id}, connection port is ${port}`); } async onPeerConnected(listener) { diff --git a/src/modules/network/network-module-manager.js b/src/modules/network/network-module-manager.js index 758ca4244a..698c7f5697 100644 --- a/src/modules/network/network-module-manager.js +++ b/src/modules/network/network-module-manager.js @@ -5,6 +5,12 @@ class NetworkModuleManager extends BaseModuleManager { return 'network'; } + async start() { + if (this.initialized) { + return this.getImplementation().module.start(); + } + } + async onPeerConnected(listener) { if (this.initialized) { return this.getImplementation().module.onPeerConnected(listener); diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index d2a00d74e2..08dc04df82 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -108,7 +108,7 @@ class ShardRepository { } async updatePeerRecordLastDialed(peerId, timestamp) { - await this.model.update( + return this.model.update( { lastDialed: timestamp, }, @@ -119,7 +119,7 @@ class ShardRepository { } async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp) { - await this.model.update( + return this.model.update( { lastDialed: timestamp, lastSeen: timestamp, diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index c5a252be41..29d63ff71a 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -488,6 +488,7 @@ class BlockchainEventListenerService { blockchain, contract, tokenId, + assertionId, ); const storePromises = []; @@ -559,12 +560,15 @@ class BlockchainEventListenerService { await Promise.all(storePromises); // remove asset from pending storage - await this.pendingStorageService.removeCachedAssertion( - pendingRepository, - blockchain, - contract, - tokenId, - ); + if (cachedData) { + await this.pendingStorageService.removeCachedAssertion( + pendingRepository, + blockchain, + contract, + tokenId, + assertionId, + ); + } } } diff --git a/src/service/file-service.js b/src/service/file-service.js index aa1c5be91b..68096b91f6 100644 --- a/src/service/file-service.js +++ b/src/service/file-service.js @@ -90,7 +90,7 @@ class FileService { } async removeFolder(folderPath) { - // this.logger.trace(`Removing folder at path: ${folderPath}`); + this.logger.trace(`Removing folder at path: ${folderPath}`); try { await rm(folderPath, { recursive: true }); @@ -146,19 +146,16 @@ class FileService { tokenId, ); - let pendingStorageFileName; - if (assertionId === undefined) { - [pendingStorageFileName] = await this.readDirectory(pendingStorageFolder); - } else { - pendingStorageFileName = assertionId; - } - - return path.join(pendingStorageFolder, pendingStorageFileName); + return path.join(pendingStorageFolder, assertionId); } getArchiveFolderPath(subFolder) { return path.join(this.getDataFolderPath(), ARCHIVE_FOLDER_NAME, subFolder); } + + getParentDirectory(filePath) { + return path.dirname(filePath); + } } export default FileService; diff --git a/src/service/pending-storage-service.js b/src/service/pending-storage-service.js index a882ce40af..0817453d27 100644 --- a/src/service/pending-storage-service.js +++ b/src/service/pending-storage-service.js @@ -34,11 +34,11 @@ class PendingStorageService { ); } - async getCachedAssertion(repository, blockchain, contract, tokenId, operationId) { + async getCachedAssertion(repository, blockchain, contract, tokenId, assertionId, operationId) { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.debug( - `Reading cached assertion for ual: ${ual}, operation id: ${operationId} from file in ${repository} pending storage`, + `Reading cached assertion for ual: ${ual}, assertion id: ${assertionId}, operation id: ${operationId} from file in ${repository} pending storage`, ); try { const documentPath = await this.fileService.getPendingStorageDocumentPath( @@ -46,30 +46,57 @@ class PendingStorageService { blockchain, contract, tokenId, + assertionId, ); const data = await this.fileService.readFile(documentPath, true); return data; } catch (error) { - this.logger.debug('Assertion not found in pending storage'); + this.logger.debug( + `Assertion not found in ${repository} pending storage. Error message: ${error.message}, ${error.stackTrace}`, + ); return null; } } - async removeCachedAssertion(repository, blockchain, contract, tokenId, operationId) { + async removeCachedAssertion( + repository, + blockchain, + contract, + tokenId, + assertionId, + operationId, + ) { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.debug( `Removing cached assertion for ual: ${ual} operation id: ${operationId} from file in ${repository} pending storage`, ); - const pendingStorageFolderPath = this.fileService.getPendingStorageFolderPath( + const pendingAssertionPath = await this.fileService.getPendingStorageDocumentPath( repository, blockchain, contract, tokenId, + assertionId, ); - await this.fileService.removeFolder(pendingStorageFolderPath); + await this.fileService.removeFile(pendingAssertionPath); + + const pendingStorageFolderPath = this.fileService.getParentDirectory(pendingAssertionPath); + + try { + const otherPendingAssertions = await this.fileService.readDirectory( + pendingStorageFolderPath, + ); + if (otherPendingAssertions.length === 0) { + await this.fileService.removeFolder(pendingStorageFolderPath); + } + } catch (error) { + this.logger.debug( + `Assertions folder not found in ${repository} pending storage. ` + + `Error message: ${error.message}, ${error.stackTrace}`, + ); + } } async assetHasPendingState(repository, blockchain, contract, tokenId, assertionId) { diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 2a09e598ef..a1200c0deb 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -152,7 +152,6 @@ class ShardingTableService { firstAssertionId, hashFunctionId, ) { - const kbSize = assertionSize < BYTES_IN_KILOBYTE ? BYTES_IN_KILOBYTE : assertionSize; const peerRecords = await this.findNeighbourhood( blockchainId, this.blockchainModuleManager.encodePacked( @@ -164,20 +163,34 @@ class ShardingTableService { hashFunctionId, true, ); - - const sorted = peerRecords.sort((a, b) => a.ask - b.ask); - - const { ask } = sorted[Math.floor(sorted.length * 0.75)]; + const r1 = await this.blockchainModuleManager.getR1(blockchainId); + // todo remove this line once we implement logic for storing assertion in publish node if it's in neighbourhood + const myPeerId = this.networkModuleManager.getPeerId().toB58String(); + const filteredPeerRecords = peerRecords.filter((peer) => peer.peerId !== myPeerId); + const sorted = filteredPeerRecords.sort((a, b) => a.ask - b.ask); + let ask; + if (sorted.length > r1) { + ask = sorted[r1 - 1].ask; + } else { + ask = sorted[sorted.length - 1].ask; + } const r0 = await this.blockchainModuleManager.getR0(blockchainId); - return this.blockchainModuleManager + const minBidSuggestion = this.blockchainModuleManager + .toBigNumber(blockchainId, '1') + .mul(epochsNumber) + .mul(r0); + + const bidSuggestion = this.blockchainModuleManager .toBigNumber(blockchainId, this.blockchainModuleManager.convertToWei(blockchainId, ask)) - .mul(kbSize) + .mul(assertionSize) .mul(epochsNumber) .mul(r0) - .div(BYTES_IN_KILOBYTE) - .toString(); + .div(BYTES_IN_KILOBYTE); + return bidSuggestion.lte(minBidSuggestion) + ? minBidSuggestion.toString() + : bidSuggestion.toString(); } async findEligibleNodes(neighbourhood, bid, r1, r0) { @@ -214,15 +227,21 @@ class ShardingTableService { }; } if (this.memoryCachedPeerIds[peerId].lastUpdated < timestampThreshold) { - await this.repositoryModuleManager.updatePeerRecordLastSeenAndLastDialed(peerId, now); - this.memoryCachedPeerIds[peerId].lastUpdated = now; + const [rowsUpdated] = + await this.repositoryModuleManager.updatePeerRecordLastSeenAndLastDialed( + peerId, + now, + ); + if (rowsUpdated) { + this.memoryCachedPeerIds[peerId].lastUpdated = now; + } } this.memoryCachedPeerIds[peerId].lastDialed = now; this.memoryCachedPeerIds[peerId].lastSeen = now; } async updatePeerRecordLastDialed(peerId) { - const now = new Date(); + const now = Date.now(); const timestampThreshold = now - PEER_RECORD_UPDATE_DELAY; if (!this.memoryCachedPeerIds[peerId]) { this.memoryCachedPeerIds[peerId] = { @@ -232,8 +251,13 @@ class ShardingTableService { }; } if (this.memoryCachedPeerIds[peerId].lastUpdated < timestampThreshold) { - await this.repositoryModuleManager.updatePeerRecordLastDialed(peerId, now); - this.memoryCachedPeerIds[peerId].lastUpdated = now; + const [rowsUpdated] = await this.repositoryModuleManager.updatePeerRecordLastDialed( + peerId, + now, + ); + if (rowsUpdated) { + this.memoryCachedPeerIds[peerId].lastUpdated = now; + } } this.memoryCachedPeerIds[peerId].lastDialed = now; } diff --git a/src/service/validation-service.js b/src/service/validation-service.js index d74700b897..a6cd284152 100644 --- a/src/service/validation-service.js +++ b/src/service/validation-service.js @@ -1,8 +1,10 @@ import { assertionMetadata } from 'assertion-tools'; +import { BYTES_IN_KILOBYTE } from '../constants/constants.js'; class ValidationService { constructor(ctx) { this.logger = ctx.logger; + this.config = ctx.config; this.validationModuleManager = ctx.validationModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; } @@ -45,6 +47,13 @@ class ValidationService { blockchain, assertionId, ); + + const blockchainAssertionSizeInKb = blockchainAssertionSize / BYTES_IN_KILOBYTE; + if (blockchainAssertionSizeInKb > this.config.maximumAssertionSizeInKb) { + throw Error( + `The size of the received assertion exceeds the maximum limit allowed.. Maximum allowed assertion size in kb: ${this.config.maximumAssertionSizeInKb}, assertion size read from blockchain in kb: ${blockchainAssertionSizeInKb}`, + ); + } const assertionSize = assertionMetadata.getAssertionSizeInBytes(assertion); if (blockchainAssertionSize !== assertionSize) { throw Error( diff --git a/test/bdd/features/get-errors.feature b/test/bdd/features/get-errors.feature index 27b8516da3..1ea6d7c22f 100644 --- a/test/bdd/features/get-errors.feature +++ b/test/bdd/features/get-errors.feature @@ -6,16 +6,16 @@ Feature: Get errors test @get-errors Scenario: Getting non-existent UAL Given I setup 4 nodes - And I wait for 2 seconds - + And I wait for 5 seconds + When I call Get directly on the node 1 with nonExistentUAL And I wait for latest resolve to finalize Then Latest Get operation finished with status: GetRouteError - + @get-errors Scenario: Getting invalid UAL Given I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Get directly on the node 1 with invalidUAL And I wait for latest resolve to finalize @@ -26,7 +26,7 @@ Feature: Get errors test Given I setup 4 nodes And I set R0 to be 1 And I set R1 to be 2 - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 1 with validAssertion And I wait for latest Publish to finalize @@ -38,7 +38,7 @@ Feature: Get errors test Given I setup 4 nodes And I set R0 to be 1 And I set R1 to be 2 - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 1 with validAssertion And I wait for latest Publish to finalize diff --git a/test/bdd/features/get.feature b/test/bdd/features/get.feature index c6e994b212..d11fcf2f30 100644 --- a/test/bdd/features/get.feature +++ b/test/bdd/features/get.feature @@ -9,7 +9,7 @@ Feature: Get asset states test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize @@ -29,7 +29,7 @@ Feature: Get asset states test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize @@ -49,7 +49,7 @@ Feature: Get asset states test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize diff --git a/test/bdd/features/publish-errors.feature b/test/bdd/features/publish-errors.feature index 47c62187fe..503f7040a3 100644 --- a/test/bdd/features/publish-errors.feature +++ b/test/bdd/features/publish-errors.feature @@ -5,8 +5,8 @@ Feature: Publish errors test @publish-errors Scenario: Publish on a node with minimum replication factor greater than the number of nodes - Given I setup 1 nodes - And I wait for 2 seconds + Given I setup 2 nodes + And I wait for 5 seconds When I call Publish on the node 1 with validAssertion And I wait for latest Publish to finalize @@ -15,7 +15,7 @@ Feature: Publish errors test @publish-errors Scenario: Publish a knowledge asset directly on the node Given I setup 1 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish directly on the node 1 with validPublishRequestBody And I wait for latest Publish to finalize diff --git a/test/bdd/features/publish.feature b/test/bdd/features/publish.feature index 0e28c9c9ed..3cdc2c54f4 100644 --- a/test/bdd/features/publish.feature +++ b/test/bdd/features/publish.feature @@ -8,7 +8,7 @@ Feature: Release related tests Given I set R0 to be 1 And I set R1 to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validAssertion And I wait for latest Publish to finalize diff --git a/test/bdd/features/update-errors.feature b/test/bdd/features/update-errors.feature index 6633eca3f2..c5e6c5659c 100644 --- a/test/bdd/features/update-errors.feature +++ b/test/bdd/features/update-errors.feature @@ -6,7 +6,7 @@ Feature: Update errors test @update-errors Scenario: Update knowledge asset that was not previously published Given I setup 1 node - And I wait for 2 seconds + And I wait for 5 seconds When I call Update directly on the node 1 with validUpdateRequestBody And I wait for latest Update to finalize diff --git a/test/bdd/features/update.feature b/test/bdd/features/update.feature index a70fd206ca..fb1d42a81e 100644 --- a/test/bdd/features/update.feature +++ b/test/bdd/features/update.feature @@ -9,7 +9,7 @@ Feature: Update asset test And I set R1 to be 2 And I set finalizationCommitsNumber to be 2 And I setup 4 nodes - And I wait for 2 seconds + And I wait for 5 seconds When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 And I wait for latest Publish to finalize diff --git a/test/bdd/steps/api/publish.mjs b/test/bdd/steps/api/publish.mjs index 87a3d5ab52..1b18aea9b7 100644 --- a/test/bdd/steps/api/publish.mjs +++ b/test/bdd/steps/api/publish.mjs @@ -91,7 +91,7 @@ When('I wait for latest Publish to finalize', { timeout: 80000 }, async function assert.fail('Unable to fetch publish result'); } // eslint-disable-next-line no-await-in-loop - await setTimeout(4000); + await setTimeout(6000); } }); diff --git a/test/bdd/steps/hooks.mjs b/test/bdd/steps/hooks.mjs index 9fbb4391c2..c1d31646b4 100644 --- a/test/bdd/steps/hooks.mjs +++ b/test/bdd/steps/hooks.mjs @@ -46,7 +46,7 @@ After(function afterMethod(testCase, done) { }); if (this.state.localBlockchain) { this.logger.info('Stopping local blockchain!'); - this.state.localBlockchain.stop(); + promises.push(this.state.localBlockchain.stop()); this.state.localBlockchain = null; } this.logger.log('After test hook, cleaning repositories'); diff --git a/test/bdd/steps/lib/local-blockchain.mjs b/test/bdd/steps/lib/local-blockchain.mjs index 65515e49eb..777238c95d 100644 --- a/test/bdd/steps/lib/local-blockchain.mjs +++ b/test/bdd/steps/lib/local-blockchain.mjs @@ -2,7 +2,7 @@ import { ethers } from 'ethers'; import { readFile } from 'fs/promises'; -import { exec } from 'child_process'; +import { exec, execSync } from 'child_process'; const Hub = JSON.parse((await readFile('node_modules/dkg-evm-module/abi/Hub.json')).toString()); const HubController = JSON.parse( @@ -20,6 +20,7 @@ const testParametersStorageParams = { minProofWindowOffsetPerc: 66, // 4 minutes maxProofWindowOffsetPerc: 66, // 4 minutes proofWindowDurationPerc: 33, // 2 minutes + updateCommitWindowDuration: 60, // 1 minute finalizationCommitsNumber: 3, }; /** @@ -92,7 +93,9 @@ class LocalBlockchain { ); } - stop() { + async stop() { + const commandLog = await execSync('npm run kill:local_blockchain'); + console.log(`Killing hardhat process: ${commandLog.toString()}`); startBlockchainProcess.kill(); } diff --git a/test/unit/mock/blockchain-module-manager-mock.js b/test/unit/mock/blockchain-module-manager-mock.js index 994f2c2d37..8236f00ec5 100644 --- a/test/unit/mock/blockchain-module-manager-mock.js +++ b/test/unit/mock/blockchain-module-manager-mock.js @@ -5,6 +5,10 @@ class BlockchainModuleManagerMock { return 20; } + getR1() { + return 8; + } + getR0() { return 3; } diff --git a/test/unit/mock/network-module-manager-mock.js b/test/unit/mock/network-module-manager-mock.js index 6c530fa249..8bc0c7b32f 100644 --- a/test/unit/mock/network-module-manager-mock.js +++ b/test/unit/mock/network-module-manager-mock.js @@ -1,3 +1,9 @@ -class NetworkModuleManagerMock {} +class NetworkModuleManagerMock { + getPeerId() { + return { + toB58String: () => 'myPeerId', + }; + } +} export default NetworkModuleManagerMock; diff --git a/test/unit/service/sharding-table-service.test.js b/test/unit/service/sharding-table-service.test.js index 373841881d..1bd0de441b 100644 --- a/test/unit/service/sharding-table-service.test.js +++ b/test/unit/service/sharding-table-service.test.js @@ -39,28 +39,56 @@ describe('Sharding table service test', async () => { expect(bidSuggestions).to.be.equal('3788323225298705400'); }); - it('Get bid suggestion, returns same token amount for size 1 Kb and size < 1 Kb', async () => { + it('Get bid suggestion, returns valid value for assertion size 1b and ask 1 wei', async () => { const epochsNumber = 5; const contentAssetStorageAddress = '0xABd59A9aa71847F499d624c492d3903dA953d67a'; const firstAssertionId = '0xb44062de45333119471934bc0340c05ff09c0b463392384bc2030cd0a20c334b'; const hashFunctionId = 1; - const bidSuggestion1Kb = await shardingTableService.getBidSuggestion( + const askInWei = '0.000000000000000001'; + const peers = shardingTableService.repositoryModuleManager.getAllPeerRecords(); + shardingTableService.repositoryModuleManager.getAllPeerRecords = () => { + peers.forEach((peer) => { + // eslint-disable-next-line no-param-reassign + peer.ask = askInWei; + }); + return peers; + }; + const bidSuggestion1B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, - BYTES_IN_KILOBYTE, + 1, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); - const bidSuggestion1B = await shardingTableService.getBidSuggestion( + expect(bidSuggestion1B).to.be.equal('15'); + const bidSuggestion10B = await shardingTableService.getBidSuggestion( 'ganache', epochsNumber, - 1, + 10, + contentAssetStorageAddress, + firstAssertionId, + hashFunctionId, + ); + expect(bidSuggestion10B).to.be.equal('15'); + const bidSuggestion1024B = await shardingTableService.getBidSuggestion( + 'ganache', + epochsNumber, + 1024, + contentAssetStorageAddress, + firstAssertionId, + hashFunctionId, + ); + expect(bidSuggestion1024B).to.be.equal('15'); + const bidSuggestion2048B = await shardingTableService.getBidSuggestion( + 'ganache', + epochsNumber, + 2048, contentAssetStorageAddress, firstAssertionId, hashFunctionId, ); - expect(bidSuggestion1B).to.be.equal(bidSuggestion1Kb); + expect(bidSuggestion2048B).to.be.equal('30'); }); }); diff --git a/test/unit/service/validation-service.test.js b/test/unit/service/validation-service.test.js index 4a5d6eb263..7cb77e625e 100644 --- a/test/unit/service/validation-service.test.js +++ b/test/unit/service/validation-service.test.js @@ -13,6 +13,9 @@ describe('Validation service test', async () => { validationModuleManager: new ValidationModuleManagerMock(), blockchainModuleManager: new BlockchainModuleManagerMock(), logger: new Logger(), + config: { + maximumAssertionSizeInKb: 2500, + }, }); });