From 6cdd879d0249981b72a2535ade2636c2fe8f7519 Mon Sep 17 00:00:00 2001 From: vterzic Date: Tue, 2 Aug 2022 14:35:23 +0200 Subject: [PATCH 01/10] Implement query feature happy path --- src/commands/query/query-command.js | 61 +++++++++++++++++++ src/constants/constants.js | 16 +++++ src/controller/http-api-router.js | 12 +++- .../v1/request-schema/query-request.js | 14 +++++ src/controller/v1/result-controller.js | 11 +--- src/controller/v1/search-controller.js | 29 ++++++++- .../migrations/20220729082627-create-query.js | 33 ++++++++++ .../implementation/sequelize/models/query.js | 21 +++++++ .../implementation/ot-triple-store.js | 34 ++++++++--- .../implementation/triple-store-constants.js | 34 +++++++---- .../triple-store-module-manager.js | 8 +-- src/service/data-service.js | 56 ++++++++++++++++- src/service/query-service.js | 34 +++++++++++ 13 files changed, 323 insertions(+), 40 deletions(-) create mode 100644 src/commands/query/query-command.js create mode 100644 src/controller/v1/request-schema/query-request.js create mode 100644 src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js create mode 100644 src/modules/repository/implementation/sequelize/models/query.js create mode 100644 src/service/query-service.js diff --git a/src/commands/query/query-command.js b/src/commands/query/query-command.js new file mode 100644 index 0000000000..efbba7bdd5 --- /dev/null +++ b/src/commands/query/query-command.js @@ -0,0 +1,61 @@ +const Command = require('../command'); +const { OPERATION_ID_STATUS } = require('../../constants/constants'); + +class QueryCommand extends Command { + constructor(ctx) { + super(ctx); + this.queryService = ctx.queryService; + } + + async execute(command) { + const { query, queryType, operationId } = command.data; + + let data; + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.QUERY.QUERY_START, + ); + try { + data = await this.queryService.query(query, queryType); + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.QUERY.QUERY_END, + ); + + await this.operationIdService.cacheOperationIdData(operationId, data); + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.COMPLETED, + ); + } catch (e) { + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.FAILED, + e.message, + ); + } + + return Command.empty(); + } + + /** + * Builds default getInitCommand + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'queryCommand', + delay: 0, + retries: 0, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +module.exports = QueryCommand; diff --git a/src/constants/constants.js b/src/constants/constants.js index 505da55ebb..829280b7ca 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -396,6 +396,13 @@ exports.OPERATION_ID_STATUS = { VALIDATING_QUERY: 'VALIDATING_QUERY', SEARCHING_ENTITIES: 'SEARCHING_ENTITIES', }, + + QUERY: { + QUERY_INIT_START: 'QUERY_INIT_START', + QUERY_INIT_END: 'QUERY_INIT_END', + QUERY_START: 'QUERY_START', + QUERY_END: 'QUERY_END', + }, }; /** @@ -445,3 +452,12 @@ exports.PUBLISH_METHOD = { PROVISION: 'PROVISION', UPDATE: 'UPDATE', }; + +/** + * Local query types + * @type {{CONSTRUCT: string, SELECT: string}} + */ +exports.QUERY_TYPES = { + SELECT: 'SELECT', + CONSTRUCT: 'CONSTRUCT', +}; diff --git a/src/controller/http-api-router.js b/src/controller/http-api-router.js index f7f1af9c29..3dceb8c719 100644 --- a/src/controller/http-api-router.js +++ b/src/controller/http-api-router.js @@ -1,5 +1,6 @@ const publishRequestSchema = require('./v1/request-schema/publish-request'); const getRequestSchema = require('./v1/request-schema/get-request'); +const queryRequestSchema = require('./v1/request-schema/query-request'); class HttpApiRouter { constructor(ctx) { @@ -28,6 +29,14 @@ class HttpApiRouter { { rateLimit: true, requestSchema: publishRequestSchema }, ); + this.httpClientModuleManager.post( + '/query', + (req, res) => { + this.searchController.handleHttpApiQueryRequest(req, res); + }, + { rateLimit: true, requestSchema: queryRequestSchema }, + ); + // this.httpClientModuleManager.post('/provision', (req, res) => { // this.publishController.handleHttpApiProvisionRequest(req, res); // }); @@ -36,9 +45,6 @@ class HttpApiRouter { // this.publishController.handleHttpApiUpdateRequest(req, res); // }); // - // this.httpClientModuleManager.post(HTTP_API_ROUTES.QUERY, (req, res) => { - // this.searchController.handleHttpApiQueryRequest(req, res); - // }); // // this.httpClientModuleManager.post(HTTP_API_ROUTES.PROOFS, (req, res) => { // this.searchController.handleHttpApiProofsRequest(req, res); diff --git a/src/controller/v1/request-schema/query-request.js b/src/controller/v1/request-schema/query-request.js new file mode 100644 index 0000000000..500cc90d76 --- /dev/null +++ b/src/controller/v1/request-schema/query-request.js @@ -0,0 +1,14 @@ +const { QUERY_TYPES } = require('../../../constants/constants'); + +module.exports = { + type: 'object', + required: ['type', 'query'], + properties: { + type: { + enum: [QUERY_TYPES.CONSTRUCT, QUERY_TYPES.SELECT], + }, + query: { + type: 'string', + }, + }, +}; diff --git a/src/controller/v1/result-controller.js b/src/controller/v1/result-controller.js index 897a852661..e76a8f9830 100644 --- a/src/controller/v1/result-controller.js +++ b/src/controller/v1/result-controller.js @@ -1,7 +1,7 @@ -const { OPERATION_ID_STATUS, ERROR_TYPE } = require('../../constants/constants'); +const { OPERATION_ID_STATUS } = require('../../constants/constants'); const BaseController = require('./base-controller'); -const availableOperations = ['publish', 'get', 'assertions:search', 'entities:search']; +const availableOperations = ['publish', 'get', 'assertions:search', 'entities:search', 'query']; class ResultController extends BaseController { constructor(ctx) { @@ -41,13 +41,8 @@ class ResultController extends BaseController { case 'assertions:search': case 'entities:search': case 'get': - if (handlerRecord.status === OPERATION_ID_STATUS.COMPLETED) { - response.data = await this.operationIdService.getCachedOperationIdData( - operationId, - ); - } - break; case 'publish': + case 'query': if (handlerRecord.status === OPERATION_ID_STATUS.COMPLETED) { response.data = await this.operationIdService.getCachedOperationIdData( operationId, diff --git a/src/controller/v1/search-controller.js b/src/controller/v1/search-controller.js index 9900e7ad8b..4d28bdfd8d 100644 --- a/src/controller/v1/search-controller.js +++ b/src/controller/v1/search-controller.js @@ -2,7 +2,6 @@ const BaseController = require('./base-controller'); const { OPERATION_ID_STATUS, NETWORK_PROTOCOLS, - ERROR_TYPE, NETWORK_MESSAGE_TYPES, } = require('../../constants/constants'); @@ -13,6 +12,7 @@ class SearchController extends BaseController { this.fileService = ctx.fileService; this.commandExecutor = ctx.commandExecutor; this.operationIdService = ctx.operationIdService; + this.queryService = ctx.queryService; } async handleHttpApiSearchAssertionsRequest(req, res) { @@ -128,7 +128,32 @@ class SearchController extends BaseController { } } - handleHttpApiQueryRequest(req, res) {} + async handleHttpApiQueryRequest(req, res) { + const { query, type: queryType } = req.body; + + const operationId = await this.operationIdService.generateOperationId( + OPERATION_ID_STATUS.QUERY.QUERY_INIT_START, + ); + + this.returnResponse(res, 202, { + operationId, + }); + + await this.operationIdService.updateOperationIdStatus( + operationId, + OPERATION_ID_STATUS.QUERY.QUERY_INIT_END, + ); + + const commandSequence = ['queryCommand']; + + await this.commandExecutor.add({ + name: commandSequence[0], + sequence: commandSequence.slice(1), + delay: 0, + data: { query, queryType, operationId }, + transactional: false, + }); + } handleHttpApiProofsRequest(req, res) {} diff --git a/src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js b/src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js new file mode 100644 index 0000000000..8e0b7e4f41 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js @@ -0,0 +1,33 @@ +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.createTable('query', { + id: { + type: Sequelize.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + operation_id: { + type: Sequelize.UUID, + allowNull: false, + }, + status: { + allowNull: false, + type: Sequelize.STRING, + }, + created_at: { + allowNull: false, + type: Sequelize.DATE, + defaultValue: Sequelize.literal('NOW()'), + }, + updated_at: { + allowNull: false, + type: Sequelize.DATE, + defaultValue: Sequelize.literal('NOW()'), + }, + }); + }, + + down: async (queryInterface) => { + await queryInterface.dropTable('resolve'); + }, +}; diff --git a/src/modules/repository/implementation/sequelize/models/query.js b/src/modules/repository/implementation/sequelize/models/query.js new file mode 100644 index 0000000000..c8ea087fe3 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/models/query.js @@ -0,0 +1,21 @@ +module.exports = (sequelize, DataTypes) => { + const query = sequelize.define( + 'query', + { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + operation_id: DataTypes.UUID, + status: DataTypes.STRING, + created_at: DataTypes.DATE, + updated_at: DataTypes.DATE, + }, + {}, + ); + query.associate = (models) => { + // associations can be defined here + }; + return query; +}; diff --git a/src/modules/triple-store/implementation/ot-triple-store.js b/src/modules/triple-store/implementation/ot-triple-store.js index dba9286637..fd6787a285 100644 --- a/src/modules/triple-store/implementation/ot-triple-store.js +++ b/src/modules/triple-store/implementation/ot-triple-store.js @@ -2,6 +2,7 @@ const Engine = require('@comunica/query-sparql').QueryEngine; const { setTimeout } = require('timers/promises'); const { SCHEMA_CONTEXT } = require('../../../constants/constants'); const constants = require('./triple-store-constants'); +const { DATA_TYPES } = require('./triple-store-constants'); class OtTripleStore { async initialize(config, logger) { @@ -74,7 +75,12 @@ class OtTripleStore { } async construct(query) { - const result = await this.executeQuery(query); + const result = await this._executeQuery(query, DATA_TYPES.N_QUADS); + return result; + } + + async select(query) { + const result = await this._executeQuery(query, DATA_TYPES.SPARQL_RESULTS_JSON); return result; } @@ -221,18 +227,28 @@ class OtTripleStore { return true; } - async executeQuery(query) { - const result = await this.queryEngine.query(query, this.queryContext); + /** + * Executes query + * @param query + * @param mediaType + * @returns {Promise} + * @private + */ + async _executeQuery(query, mediaType) { + const queryResult = await this.queryEngine.query(query, this.queryContext); const { data } = await this.queryEngine.resultToString( - result, - 'application/n-quads', + queryResult, + mediaType, this.queryContext, ); - let nquads = ''; - for await (const nquad of data) { - nquads += nquad; + + let result = ''; + + for await (const chunk of data) { + result += chunk; } - return nquads; + + return result; } async execute(query) { diff --git a/src/modules/triple-store/implementation/triple-store-constants.js b/src/modules/triple-store/implementation/triple-store-constants.js index c6174e1012..5303d65988 100644 --- a/src/modules/triple-store/implementation/triple-store-constants.js +++ b/src/modules/triple-store/implementation/triple-store-constants.js @@ -8,16 +8,26 @@ exports.DID_PREFIX = 'did:dkg'; * @constant {number} TRIPLE_STORE_CONNECT_MAX_RETRIES * - Maximum retries for connecting to triple store */ - exports.TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; +exports.TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; - /** - * @constant {number} TRIPLE_STORE_CONNECT_RETRY_FREQUENCY - * - Wait interval between retries for connecting to triple store - */ - exports.TRIPLE_STORE_CONNECT_RETRY_FREQUENCY = 10; // 10 seconds - - /** - * @constant {number} TRIPLE_STORE_QUEUE_LIMIT - * - Triple store queue limit - */ - exports.TRIPLE_STORE_QUEUE_LIMIT = 5000; +/** + * @constant {number} TRIPLE_STORE_CONNECT_RETRY_FREQUENCY + * - Wait interval between retries for connecting to triple store + */ +exports.TRIPLE_STORE_CONNECT_RETRY_FREQUENCY = 10; // 10 seconds + +/** + * @constant {number} TRIPLE_STORE_QUEUE_LIMIT + * - Triple store queue limit + */ +exports.TRIPLE_STORE_QUEUE_LIMIT = 5000; + +/** + * Triple store data types + * @type {{APPLICATION_JSON: string, N_QUADS: string, SPARQL_RESULTS_JSON: string, LD_JSON: string}} + */ +exports.DATA_TYPES = { + LD_JSON: 'application/ld+json', + N_QUADS: 'application/n-quads', + SPARQL_RESULTS_JSON: 'application/sparql-results+json', +}; diff --git a/src/modules/triple-store/triple-store-module-manager.js b/src/modules/triple-store/triple-store-module-manager.js index 745cb2aa69..9b4b13c3bd 100644 --- a/src/modules/triple-store/triple-store-module-manager.js +++ b/src/modules/triple-store/triple-store-module-manager.js @@ -51,15 +51,15 @@ class TripleStoreModuleManager extends BaseModuleManager { } } - async findAssertions(nquads) { + async select(query) { if (this.initialized) { - return this.getImplementation().module.findAssertions(nquads); + return this.getImplementation().module.select(query); } } - async select(query) { + async findAssertions(nquads) { if (this.initialized) { - return this.getImplementation().module.select(query); + return this.getImplementation().module.findAssertions(nquads); } } diff --git a/src/service/data-service.js b/src/service/data-service.js index d750bbf9bf..9d7cd3c6d5 100644 --- a/src/service/data-service.js +++ b/src/service/data-service.js @@ -1,8 +1,9 @@ const jsonld = require('jsonld'); + const { SCHEMA_CONTEXT } = require('../constants/constants'); +const { DATA_TYPES } = require('../modules/triple-store/implementation/triple-store-constants'); const ALGORITHM = 'URDNA2015'; -const FORMAT = 'application/n-quads'; class DataService { constructor(ctx) { @@ -13,7 +14,7 @@ class DataService { async toNQuads(content, inputFormat) { const options = { algorithm: ALGORITHM, - format: FORMAT, + format: DATA_TYPES.N_QUADS, }; if (inputFormat) { @@ -41,6 +42,57 @@ class DataService { return nquads; } + + /** + * Transforms bindings string to developer friendly key-value JSON + * @param bindings + * @returns {*[]} + */ + bindingsToJSON(bindings) { + const jsonBindings = JSON.parse(bindings); + return this._parseBindings(jsonBindings.results.bindings); + } + + /** + * Returns bindings in more developer friendly (key-value) form + * @param bindings + * @returns {*[]} + * @private + */ + _parseBindings(bindings) { + const json = []; + + for (const row of bindings) { + const obj = {}; + for (const columnName in row) { + obj[columnName] = this._parseBindingData(row[columnName]); + } + json.push(obj); + } + + return json; + } + + /** + * Returns cast binding value based on datatype + * @param data + * @returns {boolean|number|string} + * @private + */ + _parseBindingData(data) { + switch (data.datatype) { + case 'http://www.w3.org/2001/XMLSchema#decimal': + case 'http://www.w3.org/2001/XMLSchema#float': + case 'http://www.w3.org/2001/XMLSchema#double': + return parseFloat(data.value); + case 'http://www.w3.org/2001/XMLSchema#integer': + return parseInt(data.value, 10); + case 'http://www.w3.org/2001/XMLSchema#boolean': + return data.value === 'true'; + default: + return data.value; + } + } } module.exports = DataService; diff --git a/src/service/query-service.js b/src/service/query-service.js new file mode 100644 index 0000000000..6303cc094c --- /dev/null +++ b/src/service/query-service.js @@ -0,0 +1,34 @@ +const OperationService = require('./operation-service'); +const { OPERATION_ID_STATUS, QUERY_TYPES } = require('../constants/constants'); + +class QueryService extends OperationService { + constructor(ctx) { + super(ctx); + this.operationName = 'query'; + this.operationStatus = OPERATION_ID_STATUS.QUERY; + this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; + this.dataService = ctx.dataService; + } + + async query(query, queryType) { + switch (queryType) { + case QUERY_TYPES.CONSTRUCT: + return this.constructQuery(query); + case QUERY_TYPES.SELECT: + return this.selectQuery(query); + default: + throw new Error(`Unknown query type ${queryType}`); + } + } + + constructQuery(query) { + return this.tripleStoreModuleManager.construct(query); + } + + async selectQuery(query) { + const result = await this.tripleStoreModuleManager.select(query); + return this.dataService.bindingsToJSON(result); + } +} + +module.exports = QueryService; From b4794707a7f7a94aaa6ee4ba6202406db5b1fddf Mon Sep 17 00:00:00 2001 From: vterzic Date: Thu, 4 Aug 2022 15:55:13 +0200 Subject: [PATCH 02/10] Implement bindings fetching and binding data parsing --- package-lock.json | 10 +++- .../implementation/ot-triple-store.js | 46 +++++----------- .../implementation/triple-store-constants.js | 16 +++++- src/service/data-service.js | 54 +++++++++---------- src/service/query-service.js | 4 +- 5 files changed, 62 insertions(+), 68 deletions(-) diff --git a/package-lock.json b/package-lock.json index b7e8b78159..71f5143312 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.0-beta.1.51", + "version": "6.0.0-beta.1.52", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.0-beta.1.51", + "version": "6.0.0-beta.1.52", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.2.1", @@ -9072,6 +9072,7 @@ "resolved": "https://registry.npmjs.org/@trufflesuite/bigint-buffer/-/bigint-buffer-1.1.9.tgz", "integrity": "sha512-bdM5cEGCOhDSwminryHJbRmXc1x7dPKg6Pqns3qyTwFlxsqUgxE29lsERS3PlIW1HTjoIGMUqsk1zQQwST1Yxw==", "dev": true, + "hasInstallScript": true, "dependencies": { "node-gyp-build": "4.3.0" } @@ -9109,6 +9110,7 @@ "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.5.tgz", "integrity": "sha512-HTm14iMQKK2FjFLRTM5lAVcyaUzOnqbPtesFIvREgXpJHdQm8bWS+GkQgIkfaBYRHuCnea7w8UVNfwiAQhlr9A==", "dev": true, + "hasInstallScript": true, "optional": true, "dependencies": { "node-gyp-build": "^4.3.0" @@ -9188,6 +9190,7 @@ "resolved": "https://registry.npmjs.org/keccak/-/keccak-3.0.1.tgz", "integrity": "sha512-epq90L9jlFWCW7+pQa6JOnKn2Xgl2mtI664seYR6MHskvI9agt7AnDqmAlp9TqU4/caMYbA08Hi5DMZAl5zdkA==", "dev": true, + "hasInstallScript": true, "dependencies": { "node-addon-api": "^2.0.0", "node-gyp-build": "^4.2.0" @@ -9198,6 +9201,7 @@ "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.1.0.tgz", "integrity": "sha512-8C7oJDT44JXxh04aSSsfcMI8YiaGRhOFI9/pMEL7nWJLVsWajDPTRxsSHTM2WcTVY5nXM+SuRHzPPi0GbnDX+w==", "dev": true, + "hasInstallScript": true, "dependencies": { "abstract-leveldown": "^7.2.0", "napi-macros": "~2.0.0", @@ -9280,6 +9284,7 @@ "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-4.0.2.tgz", "integrity": "sha512-UDar4sKvWAksIlfX3xIaQReADn+WFnHvbVujpcbr+9Sf/69odMwy2MUsz5CKLQgX9nsIyrjuxL2imVyoNHa3fg==", "dev": true, + "hasInstallScript": true, "dependencies": { "elliptic": "^6.5.2", "node-addon-api": "^2.0.0", @@ -9291,6 +9296,7 @@ "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.7.tgz", "integrity": "sha512-vLt1O5Pp+flcArHGIyKEQq883nBt8nN8tVBcoL0qUXj2XT1n7p70yGIq2VK98I5FdZ1YHc0wk/koOnHjnXWk1Q==", "dev": true, + "hasInstallScript": true, "optional": true, "dependencies": { "node-gyp-build": "^4.3.0" diff --git a/src/modules/triple-store/implementation/ot-triple-store.js b/src/modules/triple-store/implementation/ot-triple-store.js index fd6787a285..10d8855954 100644 --- a/src/modules/triple-store/implementation/ot-triple-store.js +++ b/src/modules/triple-store/implementation/ot-triple-store.js @@ -2,7 +2,7 @@ const Engine = require('@comunica/query-sparql').QueryEngine; const { setTimeout } = require('timers/promises'); const { SCHEMA_CONTEXT } = require('../../../constants/constants'); const constants = require('./triple-store-constants'); -const { DATA_TYPES } = require('./triple-store-constants'); +const { MEDIA_TYPES } = require('./triple-store-constants'); class OtTripleStore { async initialize(config, logger) { @@ -75,12 +75,15 @@ class OtTripleStore { } async construct(query) { - const result = await this._executeQuery(query, DATA_TYPES.N_QUADS); + const result = await this._executeQuery(query, MEDIA_TYPES.N_QUADS); return result; } async select(query) { - const result = await this._executeQuery(query, DATA_TYPES.SPARQL_RESULTS_JSON); + // todo: add media type once bug is fixed + // no media type is passed because of comunica bug + // https://github.com/comunica/comunica/issues/1034 + const result = await this._executeQuery(query); return result; } @@ -122,7 +125,7 @@ class OtTripleStore { schema:hasIssuer ?issuer . } ORDER BY DESC(?timestamp)`; - const result = await this.execute(query); + const result = await this.select(query); return result; } @@ -134,7 +137,7 @@ class OtTripleStore { ${nquads} } }`; - let graph = await this.execute(query); + let graph = await this.select(query); graph = graph.map((x) => x.g.replace(`${constants.DID_PREFIX}:`, '')); if (graph.length && graph[0] === 'http://www.bigdata.com/rdf#nullGraph') { return []; @@ -170,7 +173,7 @@ class OtTripleStore { } ${limitQuery}`; - const result = await this.execute(sparqlQuery); + const result = await this.select(sparqlQuery); return result; } @@ -219,7 +222,7 @@ class OtTripleStore { ${limitQuery} } }`; - const result = await this.execute(sparqlQuery); + const result = await this.select(sparqlQuery); return result; } @@ -227,37 +230,16 @@ class OtTripleStore { return true; } - /** - * Executes query - * @param query - * @param mediaType - * @returns {Promise} - * @private - */ async _executeQuery(query, mediaType) { - const queryResult = await this.queryEngine.query(query, this.queryContext); - const { data } = await this.queryEngine.resultToString( - queryResult, - mediaType, - this.queryContext, - ); - - let result = ''; - - for await (const chunk of data) { - result += chunk; - } - - return result; - } - - async execute(query) { const result = await this.queryEngine.query(query, this.queryContext); - const { data } = await this.queryEngine.resultToString(result); + const { data } = await this.queryEngine.resultToString(result, mediaType); + let response = ''; + for await (const chunk of data) { response += chunk; } + return JSON.parse(response); } diff --git a/src/modules/triple-store/implementation/triple-store-constants.js b/src/modules/triple-store/implementation/triple-store-constants.js index 5303d65988..2c0213ea93 100644 --- a/src/modules/triple-store/implementation/triple-store-constants.js +++ b/src/modules/triple-store/implementation/triple-store-constants.js @@ -23,11 +23,23 @@ exports.TRIPLE_STORE_CONNECT_RETRY_FREQUENCY = 10; // 10 seconds exports.TRIPLE_STORE_QUEUE_LIMIT = 5000; /** - * Triple store data types + * Triple store media types * @type {{APPLICATION_JSON: string, N_QUADS: string, SPARQL_RESULTS_JSON: string, LD_JSON: string}} */ -exports.DATA_TYPES = { +exports.MEDIA_TYPES = { LD_JSON: 'application/ld+json', N_QUADS: 'application/n-quads', SPARQL_RESULTS_JSON: 'application/sparql-results+json', }; + +/** + * XML data types + * @type {{FLOAT: string, DECIMAL: string, DOUBLE: string, BOOLEAN: string, INTEGER: string}} + */ +exports.XML_DATA_TYPES = { + DECIMAL: 'http://www.w3.org/2001/XMLSchema#decimal', + FLOAT: 'http://www.w3.org/2001/XMLSchema#float', + DOUBLE: 'http://www.w3.org/2001/XMLSchema#double', + INTEGER: 'http://www.w3.org/2001/XMLSchema#integer', + BOOLEAN: 'http://www.w3.org/2001/XMLSchema#boolean', +}; diff --git a/src/service/data-service.js b/src/service/data-service.js index 9d7cd3c6d5..dfa838f061 100644 --- a/src/service/data-service.js +++ b/src/service/data-service.js @@ -1,7 +1,10 @@ const jsonld = require('jsonld'); const { SCHEMA_CONTEXT } = require('../constants/constants'); -const { DATA_TYPES } = require('../modules/triple-store/implementation/triple-store-constants'); +const { + MEDIA_TYPES, + XML_DATA_TYPES, +} = require('../modules/triple-store/implementation/triple-store-constants'); const ALGORITHM = 'URDNA2015'; @@ -14,7 +17,7 @@ class DataService { async toNQuads(content, inputFormat) { const options = { algorithm: ALGORITHM, - format: DATA_TYPES.N_QUADS, + format: MEDIA_TYPES.N_QUADS, }; if (inputFormat) { @@ -44,33 +47,22 @@ class DataService { } /** - * Transforms bindings string to developer friendly key-value JSON + * Returns bindings with proper data types * @param bindings * @returns {*[]} */ - bindingsToJSON(bindings) { - const jsonBindings = JSON.parse(bindings); - return this._parseBindings(jsonBindings.results.bindings); - } - - /** - * Returns bindings in more developer friendly (key-value) form - * @param bindings - * @returns {*[]} - * @private - */ - _parseBindings(bindings) { - const json = []; + parseBindings(bindings) { + const result = []; for (const row of bindings) { const obj = {}; for (const columnName in row) { - obj[columnName] = this._parseBindingData(row[columnName]); + obj[columnName] = this._parseBindingDataTypes(row[columnName]); } - json.push(obj); + result.push(obj); } - return json; + return result; } /** @@ -79,18 +71,20 @@ class DataService { * @returns {boolean|number|string} * @private */ - _parseBindingData(data) { - switch (data.datatype) { - case 'http://www.w3.org/2001/XMLSchema#decimal': - case 'http://www.w3.org/2001/XMLSchema#float': - case 'http://www.w3.org/2001/XMLSchema#double': - return parseFloat(data.value); - case 'http://www.w3.org/2001/XMLSchema#integer': - return parseInt(data.value, 10); - case 'http://www.w3.org/2001/XMLSchema#boolean': - return data.value === 'true'; + _parseBindingDataTypes(data) { + const [value, dataType] = data.split('^^'); + + switch (dataType) { + case XML_DATA_TYPES.DECIMAL: + case XML_DATA_TYPES.FLOAT: + case XML_DATA_TYPES.DOUBLE: + return parseFloat(JSON.parse(value)); + case XML_DATA_TYPES.INTEGER: + return parseInt(JSON.parse(value), 10); + case XML_DATA_TYPES.BOOLEAN: + return JSON.parse(value) === 'true'; default: - return data.value; + return value; } } } diff --git a/src/service/query-service.js b/src/service/query-service.js index 6303cc094c..eecfa73017 100644 --- a/src/service/query-service.js +++ b/src/service/query-service.js @@ -26,8 +26,8 @@ class QueryService extends OperationService { } async selectQuery(query) { - const result = await this.tripleStoreModuleManager.select(query); - return this.dataService.bindingsToJSON(result); + const bindings = await this.tripleStoreModuleManager.select(query); + return this.dataService.parseBindings(bindings); } } From 1508e1aa19522760e03fd2c3254c17484f1aebbd Mon Sep 17 00:00:00 2001 From: vterzic Date: Fri, 5 Aug 2022 07:35:38 +0200 Subject: [PATCH 03/10] Adjust ot-triple-store select and _executeQuery to be backward compatible --- src/modules/triple-store/implementation/ot-triple-store.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/triple-store/implementation/ot-triple-store.js b/src/modules/triple-store/implementation/ot-triple-store.js index 8cf012f854..6c63868aaf 100644 --- a/src/modules/triple-store/implementation/ot-triple-store.js +++ b/src/modules/triple-store/implementation/ot-triple-store.js @@ -92,7 +92,7 @@ class OtTripleStore { // no media type is passed because of comunica bug // https://github.com/comunica/comunica/issues/1034 const result = await this._executeQuery(query); - return result; + return JSON.parse(result); } async ask(query) { @@ -248,7 +248,7 @@ class OtTripleStore { response += chunk; } - return JSON.parse(response); + return response; } cleanEscapeCharacter(query) { From d304c961b5d42b831c116e830bad892b89995b19 Mon Sep 17 00:00:00 2001 From: vterzic Date: Fri, 5 Aug 2022 09:07:23 +0200 Subject: [PATCH 04/10] Remove query model and migration --- .../migrations/20220729082627-create-query.js | 33 ------------------- .../implementation/sequelize/models/query.js | 21 ------------ 2 files changed, 54 deletions(-) delete mode 100644 src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js delete mode 100644 src/modules/repository/implementation/sequelize/models/query.js diff --git a/src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js b/src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js deleted file mode 100644 index 8e0b7e4f41..0000000000 --- a/src/modules/repository/implementation/sequelize/migrations/20220729082627-create-query.js +++ /dev/null @@ -1,33 +0,0 @@ -module.exports = { - up: async (queryInterface, Sequelize) => { - await queryInterface.createTable('query', { - id: { - type: Sequelize.INTEGER, - primaryKey: true, - autoIncrement: true, - }, - operation_id: { - type: Sequelize.UUID, - allowNull: false, - }, - status: { - allowNull: false, - type: Sequelize.STRING, - }, - created_at: { - allowNull: false, - type: Sequelize.DATE, - defaultValue: Sequelize.literal('NOW()'), - }, - updated_at: { - allowNull: false, - type: Sequelize.DATE, - defaultValue: Sequelize.literal('NOW()'), - }, - }); - }, - - down: async (queryInterface) => { - await queryInterface.dropTable('resolve'); - }, -}; diff --git a/src/modules/repository/implementation/sequelize/models/query.js b/src/modules/repository/implementation/sequelize/models/query.js deleted file mode 100644 index c8ea087fe3..0000000000 --- a/src/modules/repository/implementation/sequelize/models/query.js +++ /dev/null @@ -1,21 +0,0 @@ -module.exports = (sequelize, DataTypes) => { - const query = sequelize.define( - 'query', - { - id: { - type: DataTypes.INTEGER, - primaryKey: true, - autoIncrement: true, - }, - operation_id: DataTypes.UUID, - status: DataTypes.STRING, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, - }, - {}, - ); - query.associate = (models) => { - // associations can be defined here - }; - return query; -}; From 92ffc0e6a0d38a5654ea5afd83e6f1f54909b689 Mon Sep 17 00:00:00 2001 From: vterzic Date: Wed, 24 Aug 2022 10:27:41 +0200 Subject: [PATCH 05/10] Add query schema to json-schema-service --- .husky/pre-commit | 2 +- src/controller/http-api-router.js | 2 +- src/service/json-schema-service.js | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.husky/pre-commit b/.husky/pre-commit index eb0b000614..57757f4eda 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,4 @@ #!/bin/sh . "$(dirname "$0")/_/husky.sh" -#npm run lint-staged +npm run lint-staged diff --git a/src/controller/http-api-router.js b/src/controller/http-api-router.js index 6f282464c0..5a71bc6dd5 100644 --- a/src/controller/http-api-router.js +++ b/src/controller/http-api-router.js @@ -33,7 +33,7 @@ class HttpApiRouter { (req, res) => { this.searchController.handleHttpApiQueryRequest(req, res); }, - { rateLimit: true, requestSchema: queryRequestSchema }, + { rateLimit: true, requestSchema: this.jsonSchemaService.querySchema() }, ); this.httpClientModuleManager.post( diff --git a/src/service/json-schema-service.js b/src/service/json-schema-service.js index 3720f21bd2..3244d2644a 100644 --- a/src/service/json-schema-service.js +++ b/src/service/json-schema-service.js @@ -1,6 +1,7 @@ const publishSchema = require('../controller/v1/request-schema/publish-schema'); const getSchema = require('../controller/v1/request-schema/get-schema'); const searchSchema = require('../controller/v1/request-schema/search-schema'); +const querySchema = require('../controller/v1/request-schema/query-request'); class JsonSchemaService { constructor(ctx) { @@ -18,6 +19,10 @@ class JsonSchemaService { searchSchema() { return searchSchema(); } + + querySchema() { + return querySchema; + } } module.exports = JsonSchemaService; From 7fd38e785b9c8362478ddffc2004ff68653f787b Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Wed, 24 Aug 2022 11:40:11 +0200 Subject: [PATCH 06/10] Updated errors for query command, updated super class for query service --- src/commands/query/query-command.js | 10 ++++------ src/constants/constants.js | 3 +++ src/service/query-service.js | 8 ++------ 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/commands/query/query-command.js b/src/commands/query/query-command.js index efbba7bdd5..8e7f54eb28 100644 --- a/src/commands/query/query-command.js +++ b/src/commands/query/query-command.js @@ -1,10 +1,12 @@ const Command = require('../command'); -const { OPERATION_ID_STATUS } = require('../../constants/constants'); +const { OPERATION_ID_STATUS, ERROR_TYPE } = require('../../constants/constants'); class QueryCommand extends Command { constructor(ctx) { super(ctx); this.queryService = ctx.queryService; + + this.errorType = ERROR_TYPE.QUERY.LOCAL_QUERY_ERROR; } async execute(command) { @@ -31,11 +33,7 @@ class QueryCommand extends Command { OPERATION_ID_STATUS.COMPLETED, ); } catch (e) { - await this.operationIdService.updateOperationIdStatus( - operationId, - OPERATION_ID_STATUS.FAILED, - e.message, - ); + await this.handleError(operationId, e.message, this.errorType, true); } return Command.empty(); diff --git a/src/constants/constants.js b/src/constants/constants.js index 6aae30303c..f3d93e01ad 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -173,6 +173,9 @@ exports.ERROR_TYPE = { GET_REQUEST_REMOTE_ERROR: 'GetRequestRemoteError', GET_ERROR: 'GetError', }, + QUERY: { + LOCAL_QUERY_ERROR: 'LocalQueryError', + }, }; /** * @constant {object} OPERATION_ID_STATUS - diff --git a/src/service/query-service.js b/src/service/query-service.js index eecfa73017..7e08039fb2 100644 --- a/src/service/query-service.js +++ b/src/service/query-service.js @@ -1,11 +1,7 @@ -const OperationService = require('./operation-service'); -const { OPERATION_ID_STATUS, QUERY_TYPES } = require('../constants/constants'); +const { QUERY_TYPES } = require('../constants/constants'); -class QueryService extends OperationService { +class QueryService { constructor(ctx) { - super(ctx); - this.operationName = 'query'; - this.operationStatus = OPERATION_ID_STATUS.QUERY; this.tripleStoreModuleManager = ctx.tripleStoreModuleManager; this.dataService = ctx.dataService; } From 7ade38fc1d6fa08e2add12d632a0629d157ca4d0 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 30 Aug 2022 12:21:00 +0200 Subject: [PATCH 07/10] Resolving PR comments --- src/controller/http-api-router.js | 2 +- src/controller/v1/request-schema/query-request.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/controller/http-api-router.js b/src/controller/http-api-router.js index 5a71bc6dd5..7eee3cafd7 100644 --- a/src/controller/http-api-router.js +++ b/src/controller/http-api-router.js @@ -33,7 +33,7 @@ class HttpApiRouter { (req, res) => { this.searchController.handleHttpApiQueryRequest(req, res); }, - { rateLimit: true, requestSchema: this.jsonSchemaService.querySchema() }, + { requestSchema: this.jsonSchemaService.querySchema() }, ); this.httpClientModuleManager.post( diff --git a/src/controller/v1/request-schema/query-request.js b/src/controller/v1/request-schema/query-request.js index 500cc90d76..e054cd7d99 100644 --- a/src/controller/v1/request-schema/query-request.js +++ b/src/controller/v1/request-schema/query-request.js @@ -1,6 +1,6 @@ const { QUERY_TYPES } = require('../../../constants/constants'); -module.exports = { +module.exports = () => ({ type: 'object', required: ['type', 'query'], properties: { @@ -11,4 +11,4 @@ module.exports = { type: 'string', }, }, -}; +}); From 55034927b1acc5afdc9d68d4008f46fee5dbbd50 Mon Sep 17 00:00:00 2001 From: zeroxbt Date: Tue, 30 Aug 2022 12:37:36 +0200 Subject: [PATCH 08/10] resolve PR comment --- src/service/json-schema-service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/json-schema-service.js b/src/service/json-schema-service.js index 3244d2644a..eb985644f5 100644 --- a/src/service/json-schema-service.js +++ b/src/service/json-schema-service.js @@ -21,7 +21,7 @@ class JsonSchemaService { } querySchema() { - return querySchema; + return querySchema(); } } From 32e64b34a0cc8f4a91d820c977b79244729537c3 Mon Sep 17 00:00:00 2001 From: micax3000 <73846781+micax3000@users.noreply.github.com> Date: Tue, 30 Aug 2022 12:48:02 +0200 Subject: [PATCH 09/10] Test/bdd publish errors (#2057) * getting operation status from additional nodes successfully * resolve testcase fixed. Added testcase for publishing with invalid config. * typo in test steps * refactor: extracted method in local-blockchain class * exctracted ganache keys to separate file * checking if all tests will be run now * refactor: extracted configuration setup functio * waiting for operation to finaliza uses for now * resolve changed to gge result in tests * added test tags * default node configurations are outside of bdd test steps. workflow yml file name changed Co-authored-by: djordjekovac --- .../{TEST-release.yml => TEST-bdd.yml} | 6 +- package-lock.json | 124 ++++++++++ package.json | 1 + test/bdd/features/publish-errors.feature | 49 +--- test/bdd/features/release.feature | 26 +- test/bdd/steps/api/datasets/privateKeys.json | 57 +++++ test/bdd/steps/api/publish.js | 80 +++--- test/bdd/steps/api/resolve.js | 51 ++-- test/bdd/steps/common.js | 234 +++++++++++++----- .../origintrail-test-bootstrap-config.json | 6 +- .../config/origintrail-test-node-config.json | 6 +- test/bdd/steps/lib/local-blockchain.js | 201 ++++----------- 12 files changed, 503 insertions(+), 338 deletions(-) rename .github/workflows/{TEST-release.yml => TEST-bdd.yml} (95%) create mode 100644 test/bdd/steps/api/datasets/privateKeys.json diff --git a/.github/workflows/TEST-release.yml b/.github/workflows/TEST-bdd.yml similarity index 95% rename from .github/workflows/TEST-release.yml rename to .github/workflows/TEST-bdd.yml index 051b941664..a57ff68747 100644 --- a/.github/workflows/TEST-release.yml +++ b/.github/workflows/TEST-bdd.yml @@ -1,4 +1,4 @@ -name: TEST-release +name: TEST-bdd #todo this test should be execute when opening PR to prerelease/release branches on: [pull_request] @@ -9,7 +9,7 @@ env: REPOSITORY_PASSWORD: password jobs: - test-release: + test-bdd: #todo think about locking the version - version should be the same as the one in official documentation runs-on: ubuntu-latest services: @@ -42,7 +42,7 @@ jobs: - run: sudo chmod -R 777 $ARTIFACTS_DIR - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - - run: npm run test:bdd:release; + - run: npm run test:bdd; - uses: actions/upload-artifact@v2 if: ${{ always() }} with: diff --git a/package-lock.json b/package-lock.json index a0f4e3bdf8..6884740fed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19251,6 +19251,8 @@ }, "node_modules/ganache/node_modules/@trufflesuite/bigint-buffer": { "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@trufflesuite/bigint-buffer/-/bigint-buffer-1.1.10.tgz", + "integrity": "sha512-pYIQC5EcMmID74t26GCC67946mgTJFiLXOT/BYozgrd4UEY2JHEGLhWi9cMiQCt5BSqFEvKkCHNnoj82SRjiEw==", "hasInstallScript": true, "inBundle": true, "license": "Apache-2.0", @@ -19263,6 +19265,8 @@ }, "node_modules/ganache/node_modules/@trufflesuite/bigint-buffer/node_modules/node-gyp-build": { "version": "4.4.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.4.0.tgz", + "integrity": "sha512-amJnQCcgtRVw9SvoebO3BKGESClrfXGCUTX9hSn1OuGQTQBOZmVd0Z0OlecpuRksKvbsUqALE8jls/ErClAPuQ==", "inBundle": true, "license": "MIT", "bin": { @@ -19273,6 +19277,8 @@ }, "node_modules/ganache/node_modules/@types/bn.js": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/bn.js/-/bn.js-5.1.0.tgz", + "integrity": "sha512-QSSVYj7pYFN49kW77o2s9xTCwZ8F2xLbjLLSEVh8D2F4JUhZtPAGOFLTD+ffqksBx/u4cE/KImFjyhqCjn/LIA==", "inBundle": true, "license": "MIT", "dependencies": { @@ -19281,21 +19287,29 @@ }, "node_modules/ganache/node_modules/@types/lru-cache": { "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-ssE3Vlrys7sdIzs5LOxCzTVMsU7i9oa/IaW92wF32JFb3CVczqOkru2xspuKczHEbG3nvmPY7IFqVmGGHdNbYw==", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/@types/node": { "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.0.tgz", + "integrity": "sha512-eMhwJXc931Ihh4tkU+Y7GiLzT/y/DBNpNtr4yU9O2w3SYBsr9NaOPhQlLKRmoWtI54uNwuo0IOUFQjVOTZYRvw==", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/@types/seedrandom": { "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-3.0.1.tgz", + "integrity": "sha512-giB9gzDeiCeloIXDgzFBCgjj1k4WxcDrZtGl6h1IqmUPlxF+Nx8Ve+96QCyDZ/HseB/uvDsKbpib9hU5cU53pw==", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/base64-js": { "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", "funding": [ { "type": "github", @@ -19315,11 +19329,15 @@ }, "node_modules/ganache/node_modules/brorand": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/buffer": { "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", "funding": [ { "type": "github", @@ -19355,6 +19373,8 @@ }, "node_modules/ganache/node_modules/catering": { "version": "2.1.0", + "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.0.tgz", + "integrity": "sha512-M5imwzQn6y+ODBfgi+cfgZv2hIUI6oYU/0f35Mdb1ujGeqeoI5tOnl9Q13DTH7LW+7er+NYq8stNOKZD/Z3U/A==", "inBundle": true, "license": "MIT", "dependencies": { @@ -19366,6 +19386,8 @@ }, "node_modules/ganache/node_modules/elliptic": { "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", "inBundle": true, "license": "MIT", "dependencies": { @@ -19380,11 +19402,15 @@ }, "node_modules/ganache/node_modules/elliptic/node_modules/bn.js": { "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/emittery": { "version": "0.10.0", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.10.0.tgz", + "integrity": "sha512-AGvFfs+d0JKCJQ4o01ASQLGPmSCxgfU9RFXvzPvZdjKK8oscynksuJhWrSTSw7j7Ep/sZct5b5ZhYCi8S/t0HQ==", "inBundle": true, "license": "MIT", "engines": { @@ -19396,6 +19422,8 @@ }, "node_modules/ganache/node_modules/hash.js": { "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", "inBundle": true, "license": "MIT", "dependencies": { @@ -19405,6 +19433,8 @@ }, "node_modules/ganache/node_modules/hmac-drbg": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", "inBundle": true, "license": "MIT", "dependencies": { @@ -19415,6 +19445,8 @@ }, "node_modules/ganache/node_modules/ieee754": { "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", "funding": [ { "type": "github", @@ -19434,11 +19466,15 @@ }, "node_modules/ganache/node_modules/inherits": { "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "inBundle": true, "license": "ISC" }, "node_modules/ganache/node_modules/is-buffer": { "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", "funding": [ { "type": "github", @@ -19461,6 +19497,8 @@ }, "node_modules/ganache/node_modules/keccak": { "version": "3.0.1", + "resolved": "https://registry.npmjs.org/keccak/-/keccak-3.0.1.tgz", + "integrity": "sha512-epq90L9jlFWCW7+pQa6JOnKn2Xgl2mtI664seYR6MHskvI9agt7AnDqmAlp9TqU4/caMYbA08Hi5DMZAl5zdkA==", "hasInstallScript": true, "inBundle": true, "license": "MIT", @@ -19474,6 +19512,8 @@ }, "node_modules/ganache/node_modules/leveldown": { "version": "6.1.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.1.0.tgz", + "integrity": "sha512-8C7oJDT44JXxh04aSSsfcMI8YiaGRhOFI9/pMEL7nWJLVsWajDPTRxsSHTM2WcTVY5nXM+SuRHzPPi0GbnDX+w==", "hasInstallScript": true, "inBundle": true, "license": "MIT", @@ -19488,6 +19528,8 @@ }, "node_modules/ganache/node_modules/leveldown/node_modules/abstract-leveldown": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.2.0.tgz", + "integrity": "sha512-DnhQwcFEaYsvYDnACLZhMmCWd3rkOeEvglpa4q5i/5Jlm3UIsWaxVzuXvDLFCSCWRO3yy2/+V/G7FusFgejnfQ==", "inBundle": true, "license": "MIT", "dependencies": { @@ -19504,6 +19546,8 @@ }, "node_modules/ganache/node_modules/leveldown/node_modules/level-concat-iterator": { "version": "3.1.0", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.1.0.tgz", + "integrity": "sha512-BWRCMHBxbIqPxJ8vHOvKUsaO0v1sLYZtjN3K2iZJsRBYtp+ONsY6Jfi6hy9K3+zolgQRryhIn2NRZjZnWJ9NmQ==", "inBundle": true, "license": "MIT", "dependencies": { @@ -19515,6 +19559,8 @@ }, "node_modules/ganache/node_modules/leveldown/node_modules/level-supports": { "version": "2.1.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.1.0.tgz", + "integrity": "sha512-E486g1NCjW5cF78KGPrMDRBYzPuueMZ6VBXHT6gC7A8UYWGiM14fGgp+s/L1oFfDWSPV/+SFkYCmZ0SiESkRKA==", "inBundle": true, "license": "MIT", "engines": { @@ -19523,26 +19569,36 @@ }, "node_modules/ganache/node_modules/minimalistic-assert": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", "inBundle": true, "license": "ISC" }, "node_modules/ganache/node_modules/minimalistic-crypto-utils": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/napi-macros": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/node-addon-api": { "version": "2.0.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", + "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/node-gyp-build": { "version": "4.3.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.3.0.tgz", + "integrity": "sha512-iWjXZvmboq0ja1pUGULQBexmxq8CV4xBhX7VDOTbL7ZR4FOowwY/VOtRxBN/yKxmdGoIp4j5ysNT4u3S2pDQ3Q==", "inBundle": true, "license": "MIT", "bin": { @@ -19553,6 +19609,8 @@ }, "node_modules/ganache/node_modules/queue-microtask": { "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "funding": [ { "type": "github", @@ -19572,11 +19630,15 @@ }, "node_modules/ganache/node_modules/queue-tick": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.0.tgz", + "integrity": "sha512-ULWhjjE8BmiICGn3G8+1L9wFpERNxkf8ysxkAer4+TFdRefDaXOCV5m92aMB9FtBVmn/8sETXLXY6BfW7hyaWQ==", "inBundle": true, "license": "MIT" }, "node_modules/ganache/node_modules/secp256k1": { "version": "4.0.2", + "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-4.0.2.tgz", + "integrity": "sha512-UDar4sKvWAksIlfX3xIaQReADn+WFnHvbVujpcbr+9Sf/69odMwy2MUsz5CKLQgX9nsIyrjuxL2imVyoNHa3fg==", "hasInstallScript": true, "inBundle": true, "license": "MIT", @@ -38647,6 +38709,8 @@ "dependencies": { "@trufflesuite/bigint-buffer": { "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@trufflesuite/bigint-buffer/-/bigint-buffer-1.1.10.tgz", + "integrity": "sha512-pYIQC5EcMmID74t26GCC67946mgTJFiLXOT/BYozgrd4UEY2JHEGLhWi9cMiQCt5BSqFEvKkCHNnoj82SRjiEw==", "bundled": true, "requires": { "node-gyp-build": "4.4.0" @@ -38654,12 +38718,16 @@ "dependencies": { "node-gyp-build": { "version": "4.4.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.4.0.tgz", + "integrity": "sha512-amJnQCcgtRVw9SvoebO3BKGESClrfXGCUTX9hSn1OuGQTQBOZmVd0Z0OlecpuRksKvbsUqALE8jls/ErClAPuQ==", "bundled": true } } }, "@types/bn.js": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/bn.js/-/bn.js-5.1.0.tgz", + "integrity": "sha512-QSSVYj7pYFN49kW77o2s9xTCwZ8F2xLbjLLSEVh8D2F4JUhZtPAGOFLTD+ffqksBx/u4cE/KImFjyhqCjn/LIA==", "bundled": true, "requires": { "@types/node": "*" @@ -38667,26 +38735,38 @@ }, "@types/lru-cache": { "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-ssE3Vlrys7sdIzs5LOxCzTVMsU7i9oa/IaW92wF32JFb3CVczqOkru2xspuKczHEbG3nvmPY7IFqVmGGHdNbYw==", "bundled": true }, "@types/node": { "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.0.tgz", + "integrity": "sha512-eMhwJXc931Ihh4tkU+Y7GiLzT/y/DBNpNtr4yU9O2w3SYBsr9NaOPhQlLKRmoWtI54uNwuo0IOUFQjVOTZYRvw==", "bundled": true }, "@types/seedrandom": { "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-3.0.1.tgz", + "integrity": "sha512-giB9gzDeiCeloIXDgzFBCgjj1k4WxcDrZtGl6h1IqmUPlxF+Nx8Ve+96QCyDZ/HseB/uvDsKbpib9hU5cU53pw==", "bundled": true }, "base64-js": { "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", "bundled": true }, "brorand": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", "bundled": true }, "buffer": { "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", "bundled": true, "requires": { "base64-js": "^1.3.1", @@ -38702,6 +38782,8 @@ }, "catering": { "version": "2.1.0", + "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.0.tgz", + "integrity": "sha512-M5imwzQn6y+ODBfgi+cfgZv2hIUI6oYU/0f35Mdb1ujGeqeoI5tOnl9Q13DTH7LW+7er+NYq8stNOKZD/Z3U/A==", "bundled": true, "requires": { "queue-tick": "^1.0.0" @@ -38709,6 +38791,8 @@ }, "elliptic": { "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", "bundled": true, "requires": { "bn.js": "^4.11.9", @@ -38722,16 +38806,22 @@ "dependencies": { "bn.js": { "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "bundled": true } } }, "emittery": { "version": "0.10.0", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.10.0.tgz", + "integrity": "sha512-AGvFfs+d0JKCJQ4o01ASQLGPmSCxgfU9RFXvzPvZdjKK8oscynksuJhWrSTSw7j7Ep/sZct5b5ZhYCi8S/t0HQ==", "bundled": true }, "hash.js": { "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", "bundled": true, "requires": { "inherits": "^2.0.3", @@ -38740,6 +38830,8 @@ }, "hmac-drbg": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", "bundled": true, "requires": { "hash.js": "^1.0.3", @@ -38749,18 +38841,26 @@ }, "ieee754": { "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", "bundled": true }, "inherits": { "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "bundled": true }, "is-buffer": { "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", "bundled": true }, "keccak": { "version": "3.0.1", + "resolved": "https://registry.npmjs.org/keccak/-/keccak-3.0.1.tgz", + "integrity": "sha512-epq90L9jlFWCW7+pQa6JOnKn2Xgl2mtI664seYR6MHskvI9agt7AnDqmAlp9TqU4/caMYbA08Hi5DMZAl5zdkA==", "bundled": true, "requires": { "node-addon-api": "^2.0.0", @@ -38769,6 +38869,8 @@ }, "leveldown": { "version": "6.1.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.1.0.tgz", + "integrity": "sha512-8C7oJDT44JXxh04aSSsfcMI8YiaGRhOFI9/pMEL7nWJLVsWajDPTRxsSHTM2WcTVY5nXM+SuRHzPPi0GbnDX+w==", "bundled": true, "requires": { "abstract-leveldown": "^7.2.0", @@ -38778,6 +38880,8 @@ "dependencies": { "abstract-leveldown": { "version": "7.2.0", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-7.2.0.tgz", + "integrity": "sha512-DnhQwcFEaYsvYDnACLZhMmCWd3rkOeEvglpa4q5i/5Jlm3UIsWaxVzuXvDLFCSCWRO3yy2/+V/G7FusFgejnfQ==", "bundled": true, "requires": { "buffer": "^6.0.3", @@ -38790,6 +38894,8 @@ }, "level-concat-iterator": { "version": "3.1.0", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.1.0.tgz", + "integrity": "sha512-BWRCMHBxbIqPxJ8vHOvKUsaO0v1sLYZtjN3K2iZJsRBYtp+ONsY6Jfi6hy9K3+zolgQRryhIn2NRZjZnWJ9NmQ==", "bundled": true, "requires": { "catering": "^2.1.0" @@ -38797,40 +38903,58 @@ }, "level-supports": { "version": "2.1.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.1.0.tgz", + "integrity": "sha512-E486g1NCjW5cF78KGPrMDRBYzPuueMZ6VBXHT6gC7A8UYWGiM14fGgp+s/L1oFfDWSPV/+SFkYCmZ0SiESkRKA==", "bundled": true } } }, "minimalistic-assert": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", "bundled": true }, "minimalistic-crypto-utils": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=", "bundled": true }, "napi-macros": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==", "bundled": true }, "node-addon-api": { "version": "2.0.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", + "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==", "bundled": true }, "node-gyp-build": { "version": "4.3.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.3.0.tgz", + "integrity": "sha512-iWjXZvmboq0ja1pUGULQBexmxq8CV4xBhX7VDOTbL7ZR4FOowwY/VOtRxBN/yKxmdGoIp4j5ysNT4u3S2pDQ3Q==", "bundled": true }, "queue-microtask": { "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "bundled": true }, "queue-tick": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.0.tgz", + "integrity": "sha512-ULWhjjE8BmiICGn3G8+1L9wFpERNxkf8ysxkAer4+TFdRefDaXOCV5m92aMB9FtBVmn/8sETXLXY6BfW7hyaWQ==", "bundled": true }, "secp256k1": { "version": "4.0.2", + "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-4.0.2.tgz", + "integrity": "sha512-UDar4sKvWAksIlfX3xIaQReADn+WFnHvbVujpcbr+9Sf/69odMwy2MUsz5CKLQgX9nsIyrjuxL2imVyoNHa3fg==", "bundled": true, "requires": { "elliptic": "^6.5.2", diff --git a/package.json b/package.json index 5b816e9fe8..aefebb0f7b 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "test:bdd": "cucumber-js --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ -r test/bdd/steps/", "test:unit": "nyc --all mocha --exit $(find test/unit -name '*.js')", "test:bdd:release": "cucumber-js --tags=@release --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ -r test/bdd/steps/", + "test:bdd:publish-errors": "cucumber-js --tags=@publish-errors --fail-fast --format progress --format-options '{\"colorsEnabled\": true}' test/bdd/ -r test/bdd/steps/", "lint": "eslint ." }, "repository": { diff --git a/test/bdd/features/publish-errors.feature b/test/bdd/features/publish-errors.feature index 00887cb448..7ac8d8f531 100644 --- a/test/bdd/features/publish-errors.feature +++ b/test/bdd/features/publish-errors.feature @@ -3,48 +3,19 @@ Feature: Publish errors test Given the blockchain is set up And 1 bootstrap is running -## Scenario: Node is not able to start publish on the network with valid assertion -## Given I setup 4 nodes -## When I call publish on node 1 with validAssertion -## And I wait for last publish to finalize -## And Last publish finished with status: PUBLISH_START_ERROR +#TODO: needs to be investigated; publish completes even with invalid configuration +# @publish-errors +# Scenario: Node is not able to publish on a node with invalid configuration +# Given I setup publish node 1 with invalid configuration +# Given I setup 3 nodes +# When I call publish on node 1 with validAssertion +# And Last publish finished with status: PublishStartError + ## -# + @publish-errors Scenario: Node is not able to validate assertion on the network Given I setup 4 nodes When I call publish on ot-node 1 directly with invalidPublishRequestBody - And I wait for 15 seconds and check operationId status + And I wait for last publish to finalize And Last publish finished with status: PublishValidateAssertionError -## Scenario: Node is not able to store publish result in the local database -## Given I setup 4 nodes -## When I call publish on node 1 with validAssertion -## And I wait for last publish to finalize -## And Last publish failed to store publish result with error: publishLocalStoreError -## -## Scenario: Node is not able to store publish init commands during the publish process -## Given I setup 4 nodes -## When I call publish on node 1 with validAssertion -## And I wait for last publish to finalize -## And Last publish failed to store publish init commands with error: publishStoreInitError -## -## Scenario: Node is not able to store publish request during the publish process -## Given I setup 4 nodes -## When I call publish on node 1 with validAssertion -## And I wait for last publish to finalize -## And Last publish failed to store publish request with error: publishStoreRequestError -## -## Scenario: Node is not able to find node during the publish process -## Given I setup 4 nodes -## When I call publish on node 1 with validAssertion -## And I wait for last publish to finalize -## And Last publish failed to find node with error: publishFindNodesError -## -## Scenario: Node is not able to finalized publish on the network with valid assertion -## Given I setup 4 nodes -## When I call publish on node 1 with validAssertion -## And I wait for last publish to finalize -## And Last publish failed with error: publishError -## -## -## diff --git a/test/bdd/features/release.feature b/test/bdd/features/release.feature index 7fb0dd2e20..1b9bb604e8 100644 --- a/test/bdd/features/release.feature +++ b/test/bdd/features/release.feature @@ -7,21 +7,15 @@ Feature: Release related tests Scenario: Node is able to publish on the network, expect status completed on publish result Given I setup 4 nodes When I call publish on node 1 with validAssertion - And I wait for last publish to finalize And Last publish finished with status: COMPLETED -# @release -# Scenario: Node is able to resolve assertion previously published -# Given I setup 4 nodes -# When I call publish on node 1 with validAssertion -# And I wait for last publish to finalize -# And Last publish finished with status: COMPLETED -# And I call resolve on node 1 for last published assertion -# And I wait for last resolve to finalize -# And Last resolve finished with status: COMPLETED -# And Last resolve returned valid result -## And I setup 1 additional node -## And I call resolve on node 5 for last published assertion -## And I wait for last resolve to finalize -## And Last resolve finished with status: COMPLETED -## And Last resolve returned valid result + @release + Scenario: Node is able to get result of the operation previously published + Given I setup 4 nodes + When I call publish on node 1 with validAssertion + And Last publish finished with status: COMPLETED + And I get operation result from node 1 for last published assertion + And Last operation finished with status: COMPLETED + And I setup 1 additional node + And I get operation result from node 5 for last published assertion + And Last operation finished with status: COMPLETED diff --git a/test/bdd/steps/api/datasets/privateKeys.json b/test/bdd/steps/api/datasets/privateKeys.json new file mode 100644 index 0000000000..7b52bb9809 --- /dev/null +++ b/test/bdd/steps/api/datasets/privateKeys.json @@ -0,0 +1,57 @@ +[ +"3cf97be6177acdd12796b387f58f84f177d0fe20d8558004e8db9a41cf90392a", +"1e60c8e9aa35064cd2eaa4c005bda2b76ef1a858feebb6c8e131c472d16f9740", +"2c26a937a1b8b20762e2e578899b98fd48b6ab2f8798cd03ccef2bee865c2c54", +"a76e13d35326f5e06d20655d0edb2f60b8863280fabf8e3f0b1210cf0bb72eec", +"d96876d0711ed11781efe0c04c95716c2e0acabc4eba418516d76be808a2fc54", +"6be9ea24e3c2adf0ac4c6705b445827a57c80e6fefa10df3f480da8aa2c523a4", +"6627e24e68bca3b29c548789aead92b48c0f4ce669842ff7e18ca356429b804c", +"1e3bae753e15ee5e6e2548105b53736a3db7304ac7681d00c77caca0b848b0ba", +"4bc5744a30e928d162ca580f9a034dfac7edc0415e0b9ddc2d221a155f6ec4ff", +"03c5646544ea8e47174ac98e3b97338c486860897e31333318ee62d19e5ea118", +"9fe33d5460d64c2993c687b9f2b6c3503e082388f59d0fea14142d20f805fcc5", +"843c42e809fc394e1d8647cd74edee133d5faa187933dc0fc08d302f57b6333c", +"81feca24308a669408b973dc010800d93d347db851ff9fc12ba7ec4e60846ee3", +"2fbcf9435210fed8dd47dccae453df02c0159c265dc454be8a24e4189e2a3c1b", +"5a03ebd83bf32553f00f33255154158a07020ebc2921d842a5aee2ab94370969", +"13a8fc7dac578c05cbae943f220477b2abcb9954c8cb31279fce2f864558420c", +"d35889eb100eb5544ea3e9ddab1181e628dc8e167365dcf97112fab9ae0db906", +"f261d13fb3fd1f3df2f5dc75b54066d354a25aa2800b90b42900fc0db794cc41", +"a6dc4993c4a65f78ad87cf972f468fe25c1ad86f32d479f2ad27d3c8f46a6487", +"5fc5a969744f359c109b64bb41dca7e49e1086a9298c862cd0e30772908bb70c", +"c00655521fa1326b8b1831fb74b2a5e45eca0e8e886df34e9a714ae70031c502", +"feee5c36d8a9b1c65d9b0810f048d817d6cd01f95b12e3ae8940a769e2a6d217", +"edf15b05b906d2582d269d81fe85ee64325fa081aafd64de32893d4d6b03f56d", +"92f6678cdb6ce485e305d5fa926d2d157745871fc7d72f0526048f8286f28247", +"dfa44843c22ae16de69e9181e2bfe2153e36464d266e32746de82478e9674d98", +"483af28e4e11638d018a4fa02dcb454cfff8235073921aefdb5a302956c6abb0", +"8908b19e6f8ed4aabe160719cc3cb2b15aabf79cfc436ad31574eedd2432e3bc", +"6b499a1289d1945cbc232bd47da77ae10432ffb63b7f6d04b797a74f30f22c7d", +"db908900b007ba9c384b116b6d1209d4842f37e2435d7fbd62d619643bb08e08", +"f5346004b07b6059be546f02b72a29f055251471c700e799f96529b4338ad635", +"2aa43025590ae9e9fb3eaaa75358e3a6e1719570864c43a33926a19da979ced9", +"1aa051ed6f3c40a01cad84d2c05ae3a80d897a3f7b56a88447643fc9e67cc004", +"c4505f045420e9c860989349d32a7716a9c6221c8bfc17e1012b06c4b926e530", +"35fbcc677cd348dafaa2c31519f458dcc9ddbb7278e38310e974787ca378a4a8", +"90505a5408c91fc59738f12c31f14a501c431160473819c4e7e9273092ebb288", +"e2eed5df7e6f32dfb793b7324e251950a8644d409aa194de822c1e42163e947e", +"1ac1f2db31610c84f09865b308b82d6236e09acd475b4136bd86440b7aa39c41", +"77ffe9a3e5738d8fc2fa14028e5e280872f87c7dfb5df58bd21cc6f2c7ce6f72", +"eb48615474a318cbd2e6197d01bf81d168f2d21a2a8a117bc979a887ec90d2df", +"f1a9455826b46ca2f9f66457d8faa6f02a30e1c0f0b5b5f6769b769974a5cc5f", +"afa420d816b8b97b5049ce4507b1c79ee26168bc4a197385cd848dd482746e2d", +"9fd7088936411d814238aa7072dc43c28e6ae7d298db37466dc6b3236191c3de", +"03199565ef8a1421b7fa73cbb4b4e6f4cb3470affcf8b18f783e192788d23519", +"27fa0a7dd2901067308dd9f2559204006712dc2809619a922a5fc3f43199a9b9", +"7ff5132877ee3ebaeed1664e3ff5abdcc7fb7cce57b74ce8ae2f0071e7132ab9", +"5bd9b42788ec465d52598e58857bae2b592c5b5cf8678693179a687317fe7928", +"7471241aa4a8d04058279da9266f44210a4ffd4d6ff16376ad3cab733cce3e8f", +"5ad0f83dadefdeefeee58d733ba35674f151dc1a1080bfafb9fb8778285f0646", +"966658dfe2cf0dfa999ef05ca3c926c5fe776ee4cbc7673bdea69d2907030357", +"440f37a3f0fe7560db8bc00200818c743a4a381b4d6b24967c31fc47d5ab831b", +"4072914e2feb382b79d5285d293902546938aa2b0e31cd6625ce59db77a6d3d4", +"7b39d2f9dcf59d87ca683d9797ac3f8c5ba7c9bc6ec4b6b5cfd27809160b64cb", +"455d135639bfaab54ffc729a873a1cea26e438600f0ef40642abbd2a91c9eae3", +"f04349eab3a51e2e871cbfd30d2e4c9aef791ad79b90ed32e05457b40925d8b7", +"952e45854ca5470a6d0b6cb86346c0e9c4f8f3a5a459657df8c94265183b9253" +] \ No newline at end of file diff --git a/test/bdd/steps/api/publish.js b/test/bdd/steps/api/publish.js index a9d3af9a4f..4e7093589c 100644 --- a/test/bdd/steps/api/publish.js +++ b/test/bdd/steps/api/publish.js @@ -7,9 +7,9 @@ const HttpApiHelper = require('../../../utilities/http-api-helper'); When( /^I call publish on node (\d+) with ([^"]*)/, - { timeout: 60000 }, + { timeout: 120000 }, async function publish(node, assertionName) { - this.logger.log('I call publish route successfully'); + this.logger.log(`I call publish route on node ${node}`); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, @@ -17,7 +17,7 @@ When( const { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey } = this.state.nodes[node - 1].configuration.modules.blockchain.implementation.ganache .config; - const hubContract = this.state.localBlockchain.uaiRegistryContractAddress(); + const hubContract = this.state.localBlockchain.getHubAddress(); const assertion = assertions[assertionName]; const result = await this.state.nodes[node - 1].client .publish( @@ -60,51 +60,45 @@ When( nodeId: node - 1, operationId, }; - // await setTimeout(15000); - // const status = await httpApiHelper.getOperationResult(`http://localhost:${this.state.nodes[node - 1].configuration.rpcPort}`, operationId); - // console.log(JSON.stringify(status.data,null,2)); }, ); -Given('I wait for last publish to finalize', { timeout: 60000 }, async () => { - // this.logger.log('I wait for last publish to finalize'); - // expect( - // !!this.state.lastPublishData, - // 'Last publish data is undefined. Publish is not started.', - // ).to.be.equal(true); - // const publishData = this.state.lastPublishData; - // let loopForPublishResult = true; - // let retryCount = 0; - // const maxRetryCount = 2; - // while (loopForPublishResult) { - // this.logger.log( - // `Getting publish result for operation id: ${publishData.operationId} on node: ${publishData.nodeId}`, - // ); - // // const publishResult = await httpApiHelper.getOperationResult(`http://localhost:${this.state.nodes[publishData.nodeId].configuration.rpcPort}`, publishData.operationId); - // // eslint-disable-next-line no-await-in-loop - // const publishResult = await this.state.nodes[publishData.nodeId].client - // .getResult(publishData.UAL) - // .catch((error) => { - // assert.fail(`Error while trying to get publish result assertion. ${error}`); - // }); - // if (publishResult) { - // this.state.lastPublishData.result = publishResult; - // loopForPublishResult = false; - // } - // if (retryCount === maxRetryCount) { - // loopForPublishResult = true; - // assert.fail('Unable to get publish result'); - // } else { - // retryCount += 1; - // // eslint-disable-next-line no-await-in-loop - // await setTimeout(5000); - // } - // } +Given('I wait for last publish to finalize', { timeout: 80000 }, async function publishFinalize() { + this.logger.log('I wait for last publish to finalize'); + expect( + !!this.state.lastPublishData, + 'Last publish data is undefined. Publish is not started.', + ).to.be.equal(true); + const publishData = this.state.lastPublishData; + let retryCount = 0; + const maxRetryCount = 5; + const httpApiHelper = new HttpApiHelper(); + for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { + this.logger.log( + `Getting publish result for operation id: ${publishData.operationId} on node: ${publishData.nodeId}`, + ); + // const publishResult = await httpApiHelper.getOperationResult(`http://localhost:${this.state.nodes[publishData.nodeId].configuration.rpcPort}`, publishData.operationId); + // eslint-disable-next-line no-await-in-loop + const publishResult = await httpApiHelper.getOperationResult( + this.state.nodes[publishData.nodeId].nodeRpcUrl, + publishData.operationId, + ); + this.logger.log(`Operation status: ${publishResult.data.status}`); + if (['COMPLETED', 'FAILED'].includes(publishResult.data.status)) { + this.state.lastPublishData.result = publishResult; + break; + } + if (retryCount === maxRetryCount - 1) { + assert.fail('Unable to get publish result'); + } + // eslint-disable-next-line no-await-in-loop + await setTimeout(4000); + } }); Given( - /Last publish finished with status: ([COMPLETED|FAILED|PublishValidateAssertionError]+)$/, - { timeout: 120000 }, + /Last publish finished with status: ([COMPLETED|FAILED|PublishValidateAssertionError,PUblishStartError]+)$/, + { timeout: 60000 }, async function lastPublishFinished(status) { this.logger.log(`Last publish finished with status: ${status}`); expect( @@ -125,7 +119,7 @@ Given( }, ); Given( - /I wait for (\d+) seconds and check operationId status/, + /I wait for (\d+) seconds and check operation status/, { timeout: 120000 }, async function publishWait(numberOfSeconds) { this.logger.log(`I wait for ${numberOfSeconds} seconds`); diff --git a/test/bdd/steps/api/resolve.js b/test/bdd/steps/api/resolve.js index e651326ec2..dbd74ad950 100644 --- a/test/bdd/steps/api/resolve.js +++ b/test/bdd/steps/api/resolve.js @@ -4,27 +4,33 @@ const { setTimeout } = require('timers/promises'); const sortedStringify = require('json-stable-stringify'); When( - /^I call resolve on node (\d+) for last published assertion/, + /^I get operation result from node (\d+) for last published assertion/, { timeout: 120000 }, async function resolveCall(node) { - this.logger.log('I call resolve route successfully'); + this.logger.log('I call get result for the last operation'); expect( !!this.state.lastPublishData, 'Last publish data is undefined. Publish is not finalized.', ).to.be.equal(true); - const assertionIds = [this.state.lastPublishData.result.assertion.id]; - const result = await this.state.nodes[node - 1].client - .resolve(assertionIds) - .catch((error) => { - assert.fail(`Error while trying to resolve assertion. ${error}`); - }); - const operationId = result.data.operation_id; + // const assertionIds = [this.state.lastPublishData.result.assertion.id]; - this.state.lastResolveData = { - nodeId: node - 1, - operationId, - assertionIds, - }; + // TODO: CALLING GET RESULT WITH WRONG UAL RETURNS UNDEFINED RESULT, IT SHOULD PROBABLY RETURN A FAILED RESULT MESSAGE OR SOMETHING LIKE THAT + try { + const result = await this.state.nodes[node - 1].client + .getResult(this.state.lastPublishData.UAL) + .catch((error) => { + assert.fail(`Error while trying to resolve assertion. ${error}`); + }); + const { operationId } = result.operation; + + this.state.lastResolveData = { + nodeId: node - 1, + operationId, + result, + }; + } catch (e) { + this.logger.log(`Error while getting operation result: ${e}`); + } }, ); @@ -68,23 +74,24 @@ Given( ); Given( - /Last resolve finished with status: ([COMPLETED|FAILED]+)$/, + /Last operation finished with status: ([COMPLETED|FAILED]+)$/, { timeout: 120000 }, async function lastResolveFinishedCall(status) { - this.logger.log(`Last resolve finished with status: ${status}`); + this.logger.log(`Last get result finished with status: ${status}`); expect( !!this.state.lastResolveData, - 'Last resolve data is undefined. Resolve is not started.', + 'Last get result data is undefined. Get result not started.', ).to.be.equal(true); expect( - !!this.state.lastResolveData.result, - 'Last resolve data result is undefined. Resolve is not finished.', + !!this.state.lastResolveData.result.operation, + 'Last get result data result is undefined. Get result is not finished.', ).to.be.equal(true); const resolveData = this.state.lastResolveData; - expect(resolveData.result.status, 'Publish result status validation failed').to.be.equal( - status, - ); + expect( + resolveData.result.operation.status, + 'Publish result status validation failed', + ).to.be.equal(status); }, ); diff --git a/test/bdd/steps/common.js b/test/bdd/steps/common.js index 6a5fcba649..470aaa64fc 100644 --- a/test/bdd/steps/common.js +++ b/test/bdd/steps/common.js @@ -1,12 +1,13 @@ const { Given } = require('@cucumber/cucumber'); +const DeepExtend = require('deep-extend'); const { expect, assert } = require('chai'); const { fork } = require('child_process'); const fs = require('fs'); -const path = require('path'); const DkgClientHelper = require('../../utilities/dkg-client-helper'); -const PATH_TO_CONFIGS = './config/'; const otNodeProcessPath = './test/bdd/steps/lib/ot-node-process.js'; +const defaultConfiguration = require(`./config/origintrail-test-node-config.json`); +const bootstrapNodeConfiguration = require(`./config/origintrail-test-bootstrap-config.json`); function getBlockchainConfiguration(localBlockchain, privateKey, publicKey, managementKey) { return [ { @@ -17,7 +18,7 @@ function getBlockchainConfiguration(localBlockchain, privateKey, publicKey, mana blockchainTitle: 'ganache', networkId: 'ganache::testnet', rpcEndpoints: ['http://localhost:7545'], - hubContractAddress: localBlockchain.uaiRegistryContractAddress(), + hubContractAddress: localBlockchain.getHubAddress(), evmOperationalWalletPublicKey: publicKey, evmOperationalWalletPrivateKey: privateKey, evmManagementWalletPublicKey: managementKey, @@ -34,44 +35,87 @@ function forkNode(nodeConfiguration) { return forkedNode; } -Given(/^I setup (\d+) node[s]*$/, { timeout: 60000 }, function nodeSetup(nodeCount, done) { +function createNodeConfiguration(wallet, managementWallet, nodeIndex, nodeName, rpcPort) { + return { + modules: { + blockchain: getBlockchainConfiguration( + this.state.localBlockchain, + wallet.privateKey, + wallet.address, + managementWallet.address, + )[0], + network: { + implementation: { + 'libp2p-service': { + config: { + port: 9001 + nodeIndex, + }, + }, + }, + }, + repository: { + implementation: { + 'sequelize-repository': { + config: { + database: `operationaldbnode${nodeIndex}`, + }, + }, + }, + }, + tripleStore: { + implementation: { + 'ot-graphdb': { + config: { + repository: nodeName, + }, + }, + }, + }, + httpClient: { + implementation: { + 'express-http-client': { + config: { + port: rpcPort, + }, + }, + }, + }, + }, + operationalDatabase: { + databaseName: `operationaldbnode${nodeIndex}`, + }, + rpcPort, + appDataPath: `data${nodeIndex}`, + graphDatabase: { + name: nodeName, + }, + }; +} + +Given(/^I setup (\d+) node[s]*$/, { timeout: 80000 }, function nodeSetup(nodeCount, done) { this.logger.log(`I setup ${nodeCount} node${nodeCount !== 1 ? 's' : ''}`); const wallets = this.state.localBlockchain.getWallets(); + const currentNumberOfNodes = Object.keys(this.state.nodes).length; let nodesStarted = 0; for (let i = 0; i < nodeCount; i += 1) { - const wallet = wallets[i + 1]; - const managementWallet = wallets[i + 28]; - const rpcPort = 8901 + i; - const nodeName = `origintrail-test-${i}`; + const nodeIndex = currentNumberOfNodes + i; + const wallet = wallets[nodeIndex]; + const managementWallet = wallets[nodeIndex + 28]; + const rpcPort = 8901 + nodeIndex; + const nodeName = `origintrail-test-${nodeIndex}`; - const nodeConfiguration = JSON.parse( - fs - .readFileSync( - path.join(__dirname, `${PATH_TO_CONFIGS}origintrail-test-node-config.json`), - ) - .toString(), + const nodeConfiguration = DeepExtend( + {}, + defaultConfiguration, + createNodeConfiguration.call( + this, + wallet, + managementWallet, + nodeIndex, + nodeName, + rpcPort, + ), ); - // eslint-disable-next-line prefer-destructuring - nodeConfiguration.modules.blockchain = getBlockchainConfiguration( - this.state.localBlockchain, - wallet.privateKey, - wallet.address, - managementWallet.address, - )[0]; - - nodeConfiguration.modules.network.implementation['libp2p-service'].config.port = 9001 + i; - nodeConfiguration.modules.repository.implementation[ - 'sequelize-repository' - ].config.database = `operationaldbnode${i}`; - nodeConfiguration.modules.tripleStore.implementation['ot-graphdb'].config.repository = - nodeName; - nodeConfiguration.modules.httpClient.implementation['express-http-client'].config.port = - rpcPort; - nodeConfiguration.operationalDatabase.databaseName = `operationaldbnode${i}`; - nodeConfiguration.rpcPort = rpcPort; - nodeConfiguration.appDataPath = `data${i}`; - nodeConfiguration.graphDatabase.name = nodeName; - const forkedNode = forkNode(nodeConfiguration); const logFileStream = fs.createWriteStream(`${this.state.scenarionLogDir}/${nodeName}.log`); @@ -83,7 +127,9 @@ Given(/^I setup (\d+) node[s]*$/, { timeout: 60000 }, function nodeSetup(nodeCou // eslint-disable-next-line no-loop-func forkedNode.on('message', (response) => { if (response.error) { - assert.fail(`Error while trying initialize node${i} client: ${response.error}`); + assert.fail( + `Error while trying initialize node${nodeIndex} client: ${response.error}`, + ); } else { // todo if started const client = new DkgClientHelper({ @@ -93,7 +139,7 @@ Given(/^I setup (\d+) node[s]*$/, { timeout: 60000 }, function nodeSetup(nodeCou timeout: 25, loglevel: 'trace', }); - this.state.nodes[i] = { + this.state.nodes[nodeIndex] = { client, forkedNode, configuration: nodeConfiguration, @@ -116,11 +162,6 @@ Given( expect(nodeCount).to.be.equal(1); // Currently not supported more. this.logger.log('Initializing bootstrap node'); const nodeName = 'origintrail-test-bootstrap'; - const bootstrapNodeConfiguration = JSON.parse( - fs - .readFileSync(path.join(__dirname, `${PATH_TO_CONFIGS}${nodeName}-config.json`)) - .toString(), - ); const forkedNode = forkNode(bootstrapNodeConfiguration); const logFileStream = fs.createWriteStream(`${this.state.scenarionLogDir}/${nodeName}.log`); @@ -156,7 +197,7 @@ Given( Given( /^I setup (\d+) additional node[s]*$/, - { timeout: 120000 }, + { timeout: 60000 }, function setupAdditionalNode(nodeCount, done) { this.logger.log(`I setup ${nodeCount} additional node${nodeCount !== 1 ? 's' : ''}`); const wallets = this.state.localBlockchain.getWallets(); @@ -165,48 +206,52 @@ Given( for (let i = 0; i < nodeCount; i += 1) { const nodeIndex = currentNumberOfNodes + i; const wallet = wallets[nodeIndex]; + const managementWallet = wallets[nodeIndex + 28]; const rpcPort = 8901 + nodeIndex; const nodeName = `origintrail-test-${nodeIndex}`; - const nodeConfiguration = { - graphDatabase: { - name: nodeName, - }, - blockchain: getBlockchainConfiguration( - this.state.localBlockchain, - wallet.privateKey, - wallet.address, + const nodeConfiguration = DeepExtend( + {}, + defaultConfiguration, + createNodeConfiguration.call( + this, + wallet, + managementWallet, + nodeIndex, + nodeName, + rpcPort, ), - operationalDatabase: { - databaseName: `operationaldbnode${nodeIndex}`, - }, - rpcPort, - network: { - id: 'Devnet', - port: 9001 + nodeIndex, - bootstrap: [ - '/ip4/0.0.0.0/tcp/9000/p2p/QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj', - ], - }, - }; + ); + const forkedNode = forkNode(nodeConfiguration); - const forkedNode = forkNode.call(this, nodeName, nodeConfiguration); + const logFileStream = fs.createWriteStream( + `${this.state.scenarionLogDir}/${nodeName}.log`, + ); + forkedNode.stdout.setEncoding('utf8'); + forkedNode.stdout.on('data', (data) => { + // Here is where the output goes + logFileStream.write(data); + }); // eslint-disable-next-line no-loop-func forkedNode.on('message', (response) => { if (response.error) { - // todo handle error + assert.fail( + `Error while trying initialize node${nodeIndex} client: ${response.error}`, + ); } else { // todo if started const client = new DkgClientHelper({ - endpoint: '127.0.0.1', + endpoint: 'http://localhost', port: rpcPort, useSSL: false, timeout: 25, + loglevel: 'trace', }); this.state.nodes[nodeIndex] = { client, forkedNode, configuration: nodeConfiguration, + nodeRpcUrl: `http://localhost:${rpcPort}`, }; } nodesStarted += 1; @@ -217,3 +262,60 @@ Given( } }, ); +Given( + /^I setup publish node (\d+) with invalid configuration/, + { timeout: 120000 }, + function setupPublishNode(nodeIndex, done) { + this.logger.log(`I setup node ${nodeIndex} with invalid configuration`); + const wallet = this.state.localBlockchain.getWallets()[nodeIndex - 1]; + const managementWallet = this.state.localBlockchain.getWallets()[nodeIndex - 1 + 28]; + const rpcPort = 8901 + nodeIndex - 1; + const nodeName = `origintrail-test-${nodeIndex - 1}`; + const nodeConfiguration = DeepExtend( + {}, + defaultConfiguration, + createNodeConfiguration.call( + this, + wallet, + managementWallet, + nodeIndex, + nodeName, + rpcPort, + ), + ); + nodeConfiguration.minimumAckResponses.publish = 10; + const forkedNode = forkNode(nodeConfiguration); + + const logFileStream = fs.createWriteStream(`${this.state.scenarionLogDir}/${nodeName}.log`); + forkedNode.stdout.setEncoding('utf8'); + forkedNode.stdout.on('data', (data) => { + // Here is where the output goes + logFileStream.write(data); + }); + + // eslint-disable-next-line no-loop-func + forkedNode.on('message', (response) => { + if (response.error) { + assert.fail( + `Error while trying initialize node${nodeIndex - 1} client: ${response.error}`, + ); + } else { + // todo if started + const client = new DkgClientHelper({ + endpoint: 'http://localhost', + port: rpcPort, + useSSL: false, + timeout: 25, + loglevel: 'trace', + }); + this.state.nodes[nodeIndex - 1] = { + client, + forkedNode, + configuration: nodeConfiguration, + nodeRpcUrl: `http://localhost:${rpcPort}`, + }; + } + done(); + }); + }, +); diff --git a/test/bdd/steps/config/origintrail-test-bootstrap-config.json b/test/bdd/steps/config/origintrail-test-bootstrap-config.json index 372a5919bb..2789035757 100644 --- a/test/bdd/steps/config/origintrail-test-bootstrap-config.json +++ b/test/bdd/steps/config/origintrail-test-bootstrap-config.json @@ -92,5 +92,9 @@ "name": "origintrail-test-bootstrap" }, "rpcPort": 8900, - "minimumReplicationFactor": 4 + "minimumReplicationFactor": 4, + "minimumAckResponses": { + "publish": 2, + "get": 1 + } } diff --git a/test/bdd/steps/config/origintrail-test-node-config.json b/test/bdd/steps/config/origintrail-test-node-config.json index 3d2e301744..7a1f1aa964 100644 --- a/test/bdd/steps/config/origintrail-test-node-config.json +++ b/test/bdd/steps/config/origintrail-test-node-config.json @@ -94,5 +94,9 @@ }, "rpcPort": 8901, "minimumReplicationFactor": 4, - "appDataPath": "data0" + "appDataPath": "data0", + "minimumAckResponses": { + "publish": 2, + "get": 1 + } } diff --git a/test/bdd/steps/lib/local-blockchain.js b/test/bdd/steps/lib/local-blockchain.js index 2710dc6e12..0c55c4c899 100644 --- a/test/bdd/steps/lib/local-blockchain.js +++ b/test/bdd/steps/lib/local-blockchain.js @@ -3,72 +3,32 @@ const Ganache = require('ganache-core'); const Web3 = require('web3'); const Wallet = require('ethereumjs-wallet').default; -const hubSource = require('dkg-evm-module/build/contracts/Hub.json'); -const assertionRegistrySource = require('dkg-evm-module/build/contracts/AssertionRegistry.json'); -const uaiRegistrySource = require('dkg-evm-module/build/contracts/UAIRegistry.json'); -const assetRegistrySource = require('dkg-evm-module/build/contracts/AssetRegistry.json'); -const erc20TokenSource = require('dkg-evm-module/build/contracts/ERC20Token.json'); -const profileStorageSource = require('dkg-evm-module/build/contracts/ProfileStorage.json'); -const profileSource = require('dkg-evm-module/build/contracts/Profile.json'); - -// const contractNames = ['hub','uaiRegistry','assertionRegistry','assetRegistry','erc20Token','profileStorage','profile']; - -const accountPrivateKeys = [ - '3cf97be6177acdd12796b387f58f84f177d0fe20d8558004e8db9a41cf90392a', - '1e60c8e9aa35064cd2eaa4c005bda2b76ef1a858feebb6c8e131c472d16f9740', - '2c26a937a1b8b20762e2e578899b98fd48b6ab2f8798cd03ccef2bee865c2c54', - 'a76e13d35326f5e06d20655d0edb2f60b8863280fabf8e3f0b1210cf0bb72eec', - 'd96876d0711ed11781efe0c04c95716c2e0acabc4eba418516d76be808a2fc54', - '6be9ea24e3c2adf0ac4c6705b445827a57c80e6fefa10df3f480da8aa2c523a4', - '6627e24e68bca3b29c548789aead92b48c0f4ce669842ff7e18ca356429b804c', - '1e3bae753e15ee5e6e2548105b53736a3db7304ac7681d00c77caca0b848b0ba', - '4bc5744a30e928d162ca580f9a034dfac7edc0415e0b9ddc2d221a155f6ec4ff', - '03c5646544ea8e47174ac98e3b97338c486860897e31333318ee62d19e5ea118', - '9fe33d5460d64c2993c687b9f2b6c3503e082388f59d0fea14142d20f805fcc5', - '843c42e809fc394e1d8647cd74edee133d5faa187933dc0fc08d302f57b6333c', - '81feca24308a669408b973dc010800d93d347db851ff9fc12ba7ec4e60846ee3', - '2fbcf9435210fed8dd47dccae453df02c0159c265dc454be8a24e4189e2a3c1b', - '5a03ebd83bf32553f00f33255154158a07020ebc2921d842a5aee2ab94370969', - '13a8fc7dac578c05cbae943f220477b2abcb9954c8cb31279fce2f864558420c', - 'd35889eb100eb5544ea3e9ddab1181e628dc8e167365dcf97112fab9ae0db906', - 'f261d13fb3fd1f3df2f5dc75b54066d354a25aa2800b90b42900fc0db794cc41', - 'a6dc4993c4a65f78ad87cf972f468fe25c1ad86f32d479f2ad27d3c8f46a6487', - '5fc5a969744f359c109b64bb41dca7e49e1086a9298c862cd0e30772908bb70c', - 'c00655521fa1326b8b1831fb74b2a5e45eca0e8e886df34e9a714ae70031c502', - 'feee5c36d8a9b1c65d9b0810f048d817d6cd01f95b12e3ae8940a769e2a6d217', - 'edf15b05b906d2582d269d81fe85ee64325fa081aafd64de32893d4d6b03f56d', - '92f6678cdb6ce485e305d5fa926d2d157745871fc7d72f0526048f8286f28247', - 'dfa44843c22ae16de69e9181e2bfe2153e36464d266e32746de82478e9674d98', - '483af28e4e11638d018a4fa02dcb454cfff8235073921aefdb5a302956c6abb0', - '8908b19e6f8ed4aabe160719cc3cb2b15aabf79cfc436ad31574eedd2432e3bc', - '6b499a1289d1945cbc232bd47da77ae10432ffb63b7f6d04b797a74f30f22c7d', - 'db908900b007ba9c384b116b6d1209d4842f37e2435d7fbd62d619643bb08e08', - 'f5346004b07b6059be546f02b72a29f055251471c700e799f96529b4338ad635', - '2aa43025590ae9e9fb3eaaa75358e3a6e1719570864c43a33926a19da979ced9', - '1aa051ed6f3c40a01cad84d2c05ae3a80d897a3f7b56a88447643fc9e67cc004', - 'c4505f045420e9c860989349d32a7716a9c6221c8bfc17e1012b06c4b926e530', - '35fbcc677cd348dafaa2c31519f458dcc9ddbb7278e38310e974787ca378a4a8', - '90505a5408c91fc59738f12c31f14a501c431160473819c4e7e9273092ebb288', - 'e2eed5df7e6f32dfb793b7324e251950a8644d409aa194de822c1e42163e947e', - '1ac1f2db31610c84f09865b308b82d6236e09acd475b4136bd86440b7aa39c41', - '77ffe9a3e5738d8fc2fa14028e5e280872f87c7dfb5df58bd21cc6f2c7ce6f72', - 'eb48615474a318cbd2e6197d01bf81d168f2d21a2a8a117bc979a887ec90d2df', - 'f1a9455826b46ca2f9f66457d8faa6f02a30e1c0f0b5b5f6769b769974a5cc5f', - 'afa420d816b8b97b5049ce4507b1c79ee26168bc4a197385cd848dd482746e2d', - '9fd7088936411d814238aa7072dc43c28e6ae7d298db37466dc6b3236191c3de', - '03199565ef8a1421b7fa73cbb4b4e6f4cb3470affcf8b18f783e192788d23519', - '27fa0a7dd2901067308dd9f2559204006712dc2809619a922a5fc3f43199a9b9', - '7ff5132877ee3ebaeed1664e3ff5abdcc7fb7cce57b74ce8ae2f0071e7132ab9', - '5bd9b42788ec465d52598e58857bae2b592c5b5cf8678693179a687317fe7928', - '7471241aa4a8d04058279da9266f44210a4ffd4d6ff16376ad3cab733cce3e8f', - '5ad0f83dadefdeefeee58d733ba35674f151dc1a1080bfafb9fb8778285f0646', - '966658dfe2cf0dfa999ef05ca3c926c5fe776ee4cbc7673bdea69d2907030357', - '440f37a3f0fe7560db8bc00200818c743a4a381b4d6b24967c31fc47d5ab831b', - '4072914e2feb382b79d5285d293902546938aa2b0e31cd6625ce59db77a6d3d4', - '7b39d2f9dcf59d87ca683d9797ac3f8c5ba7c9bc6ec4b6b5cfd27809160b64cb', - '455d135639bfaab54ffc729a873a1cea26e438600f0ef40642abbd2a91c9eae3', - 'f04349eab3a51e2e871cbfd30d2e4c9aef791ad79b90ed32e05457b40925d8b7', - '952e45854ca5470a6d0b6cb86346c0e9c4f8f3a5a459657df8c94265183b9253', +const sources = { + // eslint-disable-next-line global-require + hubSource: require('dkg-evm-module/build/contracts/Hub.json'), + // eslint-disable-next-line global-require + assertionRegistrySource: require('dkg-evm-module/build/contracts/AssertionRegistry.json'), + // eslint-disable-next-line global-require + uaiRegistrySource: require('dkg-evm-module/build/contracts/UAIRegistry.json'), + // eslint-disable-next-line global-require + assetRegistrySource: require('dkg-evm-module/build/contracts/AssetRegistry.json'), + // eslint-disable-next-line global-require + erc20TokenSource: require('dkg-evm-module/build/contracts/ERC20Token.json'), + // eslint-disable-next-line global-require + profileStorageSource: require('dkg-evm-module/build/contracts/ProfileStorage.json'), + // eslint-disable-next-line global-require + profileSource: require('dkg-evm-module/build/contracts/Profile.json'), +}; +const accountPrivateKeys = require('../api/datasets/privateKeys.json'); + +const contractNames = [ + 'hub', + 'uaiRegistry', + 'assertionRegistry', + 'assetRegistry', + 'erc20Token', + 'profileStorage', + 'profile', ]; const wallets = accountPrivateKeys.map((privateKey) => ({ @@ -131,7 +91,7 @@ class LocalBlockchain { ); this.fetchContracts(); await this.deployContracts(); - // this.logger.info('Contracts have been deployed!'); + this.logger.info('Contracts have been deployed!'); this.logger.info( `\t Hub contract address: \t\t\t\t\t${this.contracts.hub.instance._address}`, ); @@ -161,18 +121,9 @@ class LocalBlockchain { fetchContracts() { this.contracts = {}; - // TODO: pass contracts sources by their names - /* contractNames.forEach((name) => { - this.populateContractObject(name, `${name}Source`) - }) */ - - this.populateContractObject('hub', hubSource); - this.populateContractObject('uaiRegistry', uaiRegistrySource); - this.populateContractObject('assertionRegistry', assertionRegistrySource); - this.populateContractObject('assetRegistry', assetRegistrySource); - this.populateContractObject('erc20Token', erc20TokenSource); - this.populateContractObject('profileStorage', profileStorageSource); - this.populateContractObject('profile', profileSource); + contractNames.forEach((name) => { + this.populateContractObject(name, sources[`${name}Source`]); + }); } populateContractObject(contractName, source) { @@ -185,46 +136,21 @@ class LocalBlockchain { } async deployContracts() { - // this.logger.log('Deploying hubContract'); const deployingWallet = this.getWallets()[7]; - [this.contracts.hub.deploymentReceipt, this.contracts.hub.instance] = - await this._deployContract( - this.web3, - this.contracts.hub.artifact, - this.contracts.hub.data, - [], - deployingWallet.address, - ); + await this.deploy('hub', deployingWallet, []); await this.setContractAddress('Owner', deployingWallet.address, deployingWallet); - // this.logger.log('Deploying uaiRegistryContract'); - [this.contracts.uaiRegistry.deploymentReceipt, this.contracts.uaiRegistry.instance] = - await this._deployContract( - this.web3, - this.contracts.uaiRegistry.artifact, - this.contracts.uaiRegistry.data, - [this.contracts.hub.instance._address], - deployingWallet.address, - ); - + await this.deploy('uaiRegistry', deployingWallet, [this.contracts.hub.instance._address]); await this.setContractAddress( 'UAIRegistry', this.contracts.uaiRegistry.instance._address, deployingWallet, ); - // this.logger.log('Deploying Assertion Registry Contract'); - [ - this.contracts.assertionRegistry.deploymentReceipt, - this.contracts.assertionRegistry.instance, - ] = await this._deployContract( - this.web3, - this.contracts.assertionRegistry.artifact, - this.contracts.assertionRegistry.data, - [this.contracts.hub.instance._address], - deployingWallet.address, - ); + await this.deploy('assertionRegistry', deployingWallet, [ + this.contracts.hub.instance._address, + ]); await this.setContractAddress( 'AssertionRegistry', @@ -232,15 +158,7 @@ class LocalBlockchain { deployingWallet, ); - // this.logger.log('Deploying Asset Registry Contract'); - [this.contracts.assetRegistry.deploymentReceipt, this.contracts.assetRegistry.instance] = - await this._deployContract( - this.web3, - this.contracts.assetRegistry.artifact, - this.contracts.assetRegistry.data, - [this.contracts.hub.instance._address], - deployingWallet.address, - ); + await this.deploy('assetRegistry', deployingWallet, [this.contracts.hub.instance._address]); await this.setContractAddress( 'AssetRegistry', @@ -254,31 +172,16 @@ class LocalBlockchain { ); // this.logger.log('Deploying profileStorageContract'); - [this.contracts.profileStorage.deploymentReceipt, this.contracts.profileStorage.instance] = - await this._deployContract( - this.web3, - this.contracts.profileStorage.artifact, - this.contracts.profileStorage.data, - [this.contracts.hub.instance._address], - deployingWallet.address, - ); + await this.deploy('profileStorage', deployingWallet, [ + this.contracts.hub.instance._address, + ]); await this.setContractAddress( 'ProfileStorage', this.contracts.profileStorage.instance._address, deployingWallet, ); - // await this.contracts.profileStorage.instance.methods.setupRole() - - // this.logger.log('Deploying Token contract'); - [this.contracts.erc20Token.deploymentReceipt, this.contracts.erc20Token.instance] = - await this._deployContract( - this.web3, - this.contracts.erc20Token.artifact, - this.contracts.erc20Token.data, - [this.contracts.hub.instance._address], - deployingWallet.address, - ); + await this.deploy('erc20Token', deployingWallet, [this.contracts.hub.instance._address]); await this.setContractAddress( 'Token', @@ -287,15 +190,8 @@ class LocalBlockchain { ); await this.setupRole(this.contracts.erc20Token, deployingWallet.address); - // this.logger.log('Deploying Profile contract'); - [this.contracts.profile.deploymentReceipt, this.contracts.profile.instance] = - await this._deployContract( - this.web3, - this.contracts.profile.artifact, - this.contracts.profile.data, - [this.contracts.hub.instance._address], - deployingWallet.address, - ); + await this.deploy('profile', deployingWallet, [this.contracts.hub.instance._address]); + await this.setContractAddress( 'Profile', this.contracts.profile.instance._address, @@ -313,6 +209,17 @@ class LocalBlockchain { this.initialized = true; } + async deploy(contractName, deployingWallet, constructorArgs) { + [this.contracts[contractName].deploymentReceipt, this.contracts[contractName].instance] = + await this._deployContract( + this.web3, + this.contracts[contractName].artifact, + this.contracts[contractName].data, + constructorArgs, + deployingWallet.address, + ); + } + async _deployContract(web3, contract, contractData, constructorArguments, deployerAddress) { let deploymentReceipt; let contractInstance; @@ -365,7 +272,7 @@ class LocalBlockchain { .on('error', (error) => this.logger.error('Unable to setup role. Error: ', error)); } - uaiRegistryContractAddress() { + getHubAddress() { return this.contracts.hub.instance._address; } From de883125d416b1d3dca34598eaccf18cca9ce277 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Tue, 30 Aug 2022 13:00:20 +0200 Subject: [PATCH 10/10] Added cleaner commands (#2005) --- src/commands/command-executor.js | 33 ++++------- .../common/commands-cleaner-command.js | 57 +++++++++++++++++++ .../common/operation-id-cleaner-command.js | 54 ++++++++++++++++++ src/constants/constants.js | 30 +++++++++- .../sequelize/sequelize-repository.js | 18 ++++++ .../repository/repository-module-manager.js | 15 +++++ 6 files changed, 185 insertions(+), 22 deletions(-) create mode 100644 src/commands/common/commands-cleaner-command.js create mode 100644 src/commands/common/operation-id-cleaner-command.js diff --git a/src/commands/command-executor.js b/src/commands/command-executor.js index 62d723e178..1abef9752a 100644 --- a/src/commands/command-executor.js +++ b/src/commands/command-executor.js @@ -5,19 +5,6 @@ const { forEach } = require('p-iteration'); const Command = require('./command'); const constants = require('../constants/constants'); -/** - * Command statuses - * @type {{failed: string, expired: string, started: string, pending: string, completed: string}} - */ -const STATUS = { - failed: 'FAILED', - expired: 'EXPIRED', - started: 'STARTED', - pending: 'PENDING', - completed: 'COMPLETED', - repeating: 'REPEATING', -}; - /** * How many commands will run in parallel * @type {number} @@ -106,7 +93,7 @@ class CommandExecutor { if (command.deadline_at && now > command.deadline_at) { this.logger.warn(`Command ${command.name} and ID ${command.id} is too late...`); await this._update(command, { - status: STATUS.expired, + status: constants.COMMAND_STATUS.EXPIRED, }); try { const result = await handler.expired(command); @@ -137,7 +124,7 @@ class CommandExecutor { await this._update( command, { - status: STATUS.started, + status: constants.COMMAND_STATUS.STARTED, }, transaction, ); @@ -148,7 +135,7 @@ class CommandExecutor { await this._update( command, { - status: STATUS.repeating, + status: constants.COMMAND_STATUS.REPEATING, }, transaction, ); @@ -179,7 +166,7 @@ class CommandExecutor { await this._update( command, { - status: STATUS.completed, + status: constants.COMMAND_STATUS.COMPLETED, }, transaction, ); @@ -293,7 +280,7 @@ class CommandExecutor { if (command.retries > 1) { command.data = handler.pack(command.data); await this._update(command, { - status: STATUS.pending, + status: constants.COMMAND_STATUS.PENDING, retries: command.retries - 1, }); const period = command.period ? command.period : 0; @@ -322,7 +309,7 @@ class CommandExecutor { } else { try { await this._update(command, { - status: STATUS.failed, + status: constants.COMMAND_STATUS.FAILED, message: err.message, }); this.logger.warn(`Error in command: ${command.name}, error: ${err.message}`); @@ -359,7 +346,7 @@ class CommandExecutor { const commandInstance = this.commandResolver.resolve(command.name); command.data = commandInstance.pack(command.data); } - command.status = STATUS.pending; + command.status = constants.COMMAND_STATUS.PENDING; const opts = {}; if (transaction != null) { opts.transaction = transaction; @@ -412,7 +399,11 @@ class CommandExecutor { this.logger.info('Replay pending/started commands from the database...'); const pendingCommands = ( await this.repositoryModuleManager.getCommandsWithStatus( - [STATUS.pending, STATUS.started, STATUS.repeating], + [ + constants.COMMAND_STATUS.PENDING, + constants.COMMAND_STATUS.STARTED, + constants.COMMAND_STATUS.REPEATING, + ], ['cleanerCommand', 'autoupdaterCommand'], ) ).filter((command) => !constants.PERMANENT_COMMANDS.includes(command.name)); diff --git a/src/commands/common/commands-cleaner-command.js b/src/commands/common/commands-cleaner-command.js new file mode 100644 index 0000000000..d6326d2440 --- /dev/null +++ b/src/commands/common/commands-cleaner-command.js @@ -0,0 +1,57 @@ +const Command = require('../command'); +const { + COMMAND_STATUS, + FINALIZED_COMMAND_CLEANUP_TIME_MILLS, +} = require('../../constants/constants'); + +/** + * Increases approval for Bidding contract on blockchain + */ +class CommandsCleanerCommand extends Command { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute() { + await this.repositoryModuleManager.removeFinalizedCommands([ + COMMAND_STATUS.COMPLETED, + COMMAND_STATUS.FAILED, + COMMAND_STATUS.EXPIRED, + ]); + return Command.repeat(); + } + + /** + * Recover system from failure + * @param command + * @param error + */ + async recover(command, error) { + this.logger.warn(`Failed to clean finalized commands: error: ${error.message}`); + return Command.repeat(); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'commandsCleanerCommand', + data: {}, + period: FINALIZED_COMMAND_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +module.exports = CommandsCleanerCommand; diff --git a/src/commands/common/operation-id-cleaner-command.js b/src/commands/common/operation-id-cleaner-command.js new file mode 100644 index 0000000000..beb74a2d15 --- /dev/null +++ b/src/commands/common/operation-id-cleaner-command.js @@ -0,0 +1,54 @@ +const Command = require('../command'); +const constants = require('../../constants/constants'); + +/** + * Increases approval for Bidding contract on blockchain + */ +class OperationIdCleanerCommand extends Command { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute() { + const timeToBeDeleted = Date.now() - constants.OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS; + await this.repositoryModuleManager.removeOperationIdRecord(timeToBeDeleted, [ + constants.OPERATION_ID_STATUS.COMPLETED, + constants.OPERATION_ID_STATUS.FAILED, + ]); + return Command.repeat(); + } + + /** + * Recover system from failure + * @param command + * @param error + */ + async recover(command, error) { + this.logger.warn(`Failed to clean operation ids table: error: ${error.message}`); + return Command.repeat(); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'operationIdCleanerCommand', + period: constants.OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, + data: {}, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +module.exports = OperationIdCleanerCommand; diff --git a/src/constants/constants.js b/src/constants/constants.js index f3d93e01ad..fc1b447c66 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -79,7 +79,12 @@ exports.OPERATION_IDS_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; /** * @constant {Array} PERMANENT_COMMANDS - List of all permanent commands */ -exports.PERMANENT_COMMANDS = ['otnodeUpdateCommand', 'sendTelemetryCommand']; +exports.PERMANENT_COMMANDS = [ + 'otnodeUpdateCommand', + 'sendTelemetryCommand', + 'operationIdCleanerCommand', + 'commandsCleanerCommand', +]; /** * @constant {number} MAX_COMMAND_DELAY_IN_MILLS - Maximum delay for commands @@ -248,6 +253,29 @@ exports.OPERATIONS = { SEARCH: 'search', }; +/** + * @constant {number} OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS - + * operation id command cleanup interval time 24h + */ +exports.OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; +/** + * @constant {number} FINALIZED_COMMAND_CLEANUP_TIME_MILLS - Command cleanup interval time + * finalized commands command cleanup interval time 24h + */ +exports.FINALIZED_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; +/** + * @constant {number} COMMAND_STATUS - + * Status for commands + */ +exports.COMMAND_STATUS = { + FAILED: 'FAILED', + EXPIRED: 'EXPIRED', + STARTED: 'STARTED', + PENDING: 'PENDING', + COMPLETED: 'COMPLETED', + REPEATING: 'REPEATING', +}; + /** * @constant {object} NETWORK_PROTOCOLS - * Network protocols diff --git a/src/modules/repository/implementation/sequelize/sequelize-repository.js b/src/modules/repository/implementation/sequelize/sequelize-repository.js index f9038100bf..6566f47cd4 100644 --- a/src/modules/repository/implementation/sequelize/sequelize-repository.js +++ b/src/modules/repository/implementation/sequelize/sequelize-repository.js @@ -141,6 +141,15 @@ class SequelizeRepository { }); } + async removeFinalizedCommands(finalizedStatuses) { + await this.models.commands.destroy({ + where: { + status: { [Sequelize.Op.in]: finalizedStatuses }, + started_at: { [Sequelize.Op.lte]: Date.now() }, + }, + }); + } + // OPERATION_ID async createOperationIdRecord(handlerData) { const handlerRecord = await this.models.operation_ids.create(handlerData); @@ -164,6 +173,15 @@ class SequelizeRepository { }); } + async removeOperationIdRecord(timeToBeDeleted, statuses) { + await this.models.operation_ids.destroy({ + where: { + timestamp: { [Sequelize.Op.lt]: timeToBeDeleted }, + status: { [Sequelize.Op.in]: statuses }, + }, + }); + } + async getNumberOfNodesFoundForPublish(publishId) { return this.models.publish.findOne({ attributes: ['nodes_found'], diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 1c217d5412..77537eba47 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -45,6 +45,12 @@ class RepositoryModuleManager extends BaseModuleManager { } } + async removeFinalizedCommands(finalizedStatuses) { + if (this.initialized) { + return this.getImplementation().module.removeFinalizedCommands(finalizedStatuses); + } + } + // OPERATION ID TABLE async createOperationIdRecord(handlerData) { if (this.initialized) { @@ -64,6 +70,15 @@ class RepositoryModuleManager extends BaseModuleManager { } } + async removeOperationIdRecord(timeToBeDeleted, statuses) { + if (this.initialized) { + return this.getImplementation().module.removeOperationIdRecord( + timeToBeDeleted, + statuses, + ); + } + } + // publish table async createOperationRecord(operation, operationId, status) { if (this.initialized) {