From 243a76880b4272fd45f169508a46f449f4299906 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 25 Jun 2020 13:19:19 +0200 Subject: [PATCH 1/9] sort routing messages --- modules/Utilities.js | 16 ++++++++++++++-- .../dh/dh-data-read-request-free-command.js | 2 +- .../command/dh/dh-purchase-requested-command.js | 2 +- .../dh/dh-read-data-location-request-command.js | 2 +- .../command/dv/dv-data-read-request-command.js | 2 +- .../dv-permissioned-data-read-request-command.js | 2 +- .../command/dv/dv-purchase-request-command.js | 2 +- modules/command/dv/dv-query-network-command.js | 2 +- modules/controller/dc-controller.js | 2 +- modules/controller/dv-controller.js | 2 +- modules/service/dh-service.js | 4 ++-- 11 files changed, 25 insertions(+), 13 deletions(-) diff --git a/modules/Utilities.js b/modules/Utilities.js index aeca2a9ed4..7067ca6e3a 100644 --- a/modules/Utilities.js +++ b/modules/Utilities.js @@ -686,8 +686,9 @@ class Utilities { } static generateRsvSignature(message, web3, privateKey) { + const sortedMessage = JSON.stringify(Utilities.sortObject(message)); const signature = web3.eth.accounts.sign( - message, + sortedMessage, privateKey.toLowerCase().startsWith('0x') ? privateKey : `0x${privateKey}`, ); @@ -703,7 +704,18 @@ class Utilities { signature.s, ); - return Utilities.compareHexStrings(signedAddress, message.wallet); + // todo remove this patch in the next release + if (!Utilities.compareHexStrings(signedAddress, message.wallet)) { + const sortedMessage = Utilities.sortObject(message); + const signedAddress = web3.eth.accounts.recover( + JSON.stringify(sortedMessage), + signature.v, + signature.r, + signature.s, + ); + return Utilities.compareHexStrings(signedAddress, message.wallet); + } + return true; } /** diff --git a/modules/command/dh/dh-data-read-request-free-command.js b/modules/command/dh/dh-data-read-request-free-command.js index a4d55540ba..3ef9954818 100644 --- a/modules/command/dh/dh-data-read-request-free-command.js +++ b/modules/command/dh/dh-data-read-request-free-command.js @@ -87,7 +87,7 @@ class DHDataReadRequestFreeCommand extends Command { const dataReadResponseObject = { message: replyMessage, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(replyMessage), + replyMessage, this.web3, this.config.node_private_key, ), diff --git a/modules/command/dh/dh-purchase-requested-command.js b/modules/command/dh/dh-purchase-requested-command.js index b8732390d4..e092bf451b 100644 --- a/modules/command/dh/dh-purchase-requested-command.js +++ b/modules/command/dh/dh-purchase-requested-command.js @@ -122,7 +122,7 @@ class DhPurchaseRequestedCommand extends Command { const dataPurchaseResponseObject = { message: response, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(response), + response, this.web3, this.config.node_private_key, ), diff --git a/modules/command/dh/dh-read-data-location-request-command.js b/modules/command/dh/dh-read-data-location-request-command.js index 42e381a9d5..c2a5ce0393 100644 --- a/modules/command/dh/dh-read-data-location-request-command.js +++ b/modules/command/dh/dh-read-data-location-request-command.js @@ -180,7 +180,7 @@ class DHReadDataLocationRequestCommand extends Command { const messageResponseSignature = Utilities.generateRsvSignature( - JSON.stringify(messageResponse), + messageResponse, this.web3, this.config.node_private_key, ); diff --git a/modules/command/dv/dv-data-read-request-command.js b/modules/command/dv/dv-data-read-request-command.js index b08b1abcd2..45e9a0387c 100644 --- a/modules/command/dv/dv-data-read-request-command.js +++ b/modules/command/dv/dv-data-read-request-command.js @@ -35,7 +35,7 @@ class DVDataReadRequestCommand extends Command { const dataReadRequestObject = { message, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(message), + message, this.web3, this.config.node_private_key, ), diff --git a/modules/command/dv/dv-permissioned-data-read-request-command.js b/modules/command/dv/dv-permissioned-data-read-request-command.js index cd7da7a803..b9a19df6b2 100644 --- a/modules/command/dv/dv-permissioned-data-read-request-command.js +++ b/modules/command/dv/dv-permissioned-data-read-request-command.js @@ -39,7 +39,7 @@ class DVPermissionedDataReadRequestCommand extends Command { const dataReadRequestObject = { message, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(message), + message, this.web3, this.config.node_private_key, ), diff --git a/modules/command/dv/dv-purchase-request-command.js b/modules/command/dv/dv-purchase-request-command.js index 29ab64c56f..36e62ba8dc 100644 --- a/modules/command/dv/dv-purchase-request-command.js +++ b/modules/command/dv/dv-purchase-request-command.js @@ -87,7 +87,7 @@ class DvPurchaseRequestCommand extends Command { const dataPurchaseRequestObject = { message, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(message), + message, this.web3, this.config.node_private_key, ), diff --git a/modules/command/dv/dv-query-network-command.js b/modules/command/dv/dv-query-network-command.js index 8e9b754b55..3fbb3756ce 100644 --- a/modules/command/dv/dv-query-network-command.js +++ b/modules/command/dv/dv-query-network-command.js @@ -42,7 +42,7 @@ class DVQueryNetworkCommand extends Command { dataLocationRequestObject.messageSignature = Utilities.generateRsvSignature( - JSON.stringify(dataLocationRequestObject.message), + dataLocationRequestObject.message, this.web3, this.config.node_private_key, ); diff --git a/modules/controller/dc-controller.js b/modules/controller/dc-controller.js index 7dcfd9173f..88e04388bf 100644 --- a/modules/controller/dc-controller.js +++ b/modules/controller/dc-controller.js @@ -289,7 +289,7 @@ class DCController { const dataPriceResponseObject = { message: response, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(response), + response, this.web3, this.config.node_private_key, ), diff --git a/modules/controller/dv-controller.js b/modules/controller/dv-controller.js index 0dc5567521..4a36fc354f 100644 --- a/modules/controller/dv-controller.js +++ b/modules/controller/dv-controller.js @@ -331,7 +331,7 @@ class DVController { const dataPriceRequestObject = { message, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(message), + message, this.web3, this.config.node_private_key, ), diff --git a/modules/service/dh-service.js b/modules/service/dh-service.js index cee5dea49e..4a47d80cde 100644 --- a/modules/service/dh-service.js +++ b/modules/service/dh-service.js @@ -521,7 +521,7 @@ class DHService { const dataReadResponseObject = { message: replyMessage, messageSignature: Utilities.generateRsvSignature( - JSON.stringify(replyMessage), + replyMessage, this.web3, this.config.node_private_key, ), @@ -663,7 +663,7 @@ class DHService { }, }; encryptedPaddedKeyObject.messageSignature = Utilities.generateRsvSignature( - JSON.stringify(encryptedPaddedKeyObject.message), + encryptedPaddedKeyObject.message, this.web3, this.config.node_private_key, ); From 840adc5e32785d72ff5c36d698d05b230fc300fd Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Fri, 26 Jun 2020 10:56:55 +0200 Subject: [PATCH 2/9] added check for string messages --- modules/Utilities.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/Utilities.js b/modules/Utilities.js index 7067ca6e3a..ff819c03bd 100644 --- a/modules/Utilities.js +++ b/modules/Utilities.js @@ -686,7 +686,12 @@ class Utilities { } static generateRsvSignature(message, web3, privateKey) { - const sortedMessage = JSON.stringify(Utilities.sortObject(message)); + let sortedMessage; + if (typeof message === 'string' || message instanceof String) { + sortedMessage = message; + } else { + sortedMessage = JSON.stringify(Utilities.sortObject(message)); + } const signature = web3.eth.accounts.sign( sortedMessage, privateKey.toLowerCase().startsWith('0x') ? @@ -697,8 +702,14 @@ class Utilities { } static isMessageSigned(web3, message, signature) { + let sortedMessage; + if (typeof message === 'string' || message instanceof String) { + sortedMessage = message; + } else { + sortedMessage = JSON.stringify(message); + } const signedAddress = web3.eth.accounts.recover( - JSON.stringify(message), + sortedMessage, signature.v, signature.r, signature.s, From 285d2c407b67114d7bf976af133c8c20fe7ff66a Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Fri, 26 Jun 2020 13:44:34 +0200 Subject: [PATCH 3/9] version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4b93f31a24..515710e4ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "4.1.5", + "version": "4.1.6", "description": "OriginTrail node", "main": ".eslintrc.js", "config": { From 415998debb11ade0d79357d057bdc69ba9bd8f39 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Mon, 29 Jun 2020 16:33:06 +0200 Subject: [PATCH 4/9] added support for ot-json version 1.2 --- importers/use_cases/otjson_1.1/sort1.json | 202 ++++++++++++++++++++++ importers/use_cases/otjson_1.1/sort2.json | 202 ++++++++++++++++++++++ importers/use_cases/otjson_1.2/sort1.json | 202 ++++++++++++++++++++++ importers/use_cases/otjson_1.2/sort2.json | 202 ++++++++++++++++++++++ modules/ImportUtilities.js | 30 ++-- modules/OtJsonUtilities.js | 31 +++- modules/Utilities.js | 13 +- modules/service/import-service.js | 2 +- test/bdd/features/importer.feature | 43 ++++- test/bdd/steps/network.js | 74 ++++++++ 10 files changed, 976 insertions(+), 25 deletions(-) create mode 100644 importers/use_cases/otjson_1.1/sort1.json create mode 100644 importers/use_cases/otjson_1.1/sort2.json create mode 100644 importers/use_cases/otjson_1.2/sort1.json create mode 100644 importers/use_cases/otjson_1.2/sort2.json diff --git a/importers/use_cases/otjson_1.1/sort1.json b/importers/use_cases/otjson_1.1/sort1.json new file mode 100644 index 0000000000..f983e04de8 --- /dev/null +++ b/importers/use_cases/otjson_1.1/sort1.json @@ -0,0 +1,202 @@ +{ + "@id": "", + "@type": "Dataset", + "datasetHeader": { + "OTJSONVersion": "1.1", + "datasetCreationTimestamp": "2020-06-29T11:01:09.035Z", + "datasetTitle": "", + "datasetDescription": "", + "datasetTags": [], + "relatedDatasets": [], + "validationSchemas": { + "erc725-main": { + "schemaType": "ethereum-725", + "networkId": "ganache" + }, + "merkleRoot": { + "schemaType": "merkle-root", + "networkId": "ganache", + "hubContractAddress": "0x0987197628Bb06133B6FA2409eb4cF9FCaFe8d3a" + } + }, + "dataIntegrity": { + "proofs": [ + { + "proofValue": "", + "proofType": "merkleRootHash", + "validationSchema": "/schemas/merkleRoot" + } + ] + }, + "dataCreator": { + "identifiers": [ + { + "identifierValue": "0x2Fa6d32E314AAB43a58999D6f5532A15418Da153", + "identifierType": "ERC725", + "validationSchema": "/schemas/erc725-main" + } + ] + } + }, + "@graph": [ + { + "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 3, + "factory_name": "DONG CITY TESTING NEW AUDITT", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Reviewed", + "audit_guid": "Final_test_scan", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "10.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + } + ] + }, + "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + } + }, + "relations": [] + }, + { + "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 1, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pendind Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "version": 2, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pending Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Pendind Review", + "version": 1, + "audit_status_update_time": "2020-06-15 10:28:08" + }, + { + "audit_status": "Audit Pending Review", + "version": 2, + "audit_status_update_time": "2020-06-15 10:32:37" + } + ] + }, + "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + } + }, + "relations": [] + } + ] +} \ No newline at end of file diff --git a/importers/use_cases/otjson_1.1/sort2.json b/importers/use_cases/otjson_1.1/sort2.json new file mode 100644 index 0000000000..f00ba6f876 --- /dev/null +++ b/importers/use_cases/otjson_1.1/sort2.json @@ -0,0 +1,202 @@ +{ + "@id": "", + "@type": "Dataset", + "datasetHeader": { + "OTJSONVersion": "1.1", + "datasetCreationTimestamp": "2020-06-29T11:01:09.035Z", + "datasetTitle": "", + "datasetDescription": "", + "datasetTags": [], + "relatedDatasets": [], + "validationSchemas": { + "erc725-main": { + "schemaType": "ethereum-725", + "networkId": "ganache" + }, + "merkleRoot": { + "schemaType": "merkle-root", + "networkId": "ganache", + "hubContractAddress": "0x0987197628Bb06133B6FA2409eb4cF9FCaFe8d3a" + } + }, + "dataIntegrity": { + "proofs": [ + { + "proofValue": "", + "proofType": "merkleRootHash", + "validationSchema": "/schemas/merkleRoot" + } + ] + }, + "dataCreator": { + "identifiers": [ + { + "identifierValue": "0x2Fa6d32E314AAB43a58999D6f5532A15418Da153", + "identifierType": "ERC725", + "validationSchema": "/schemas/erc725-main" + } + ] + } + }, + "@graph": [ + { + "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 1, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pendind Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "version": 2, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pending Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Pendind Review", + "version": 1, + "audit_status_update_time": "2020-06-15 10:28:08" + }, + { + "audit_status": "Audit Pending Review", + "version": 2, + "audit_status_update_time": "2020-06-15 10:32:37" + } + ] + }, + "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + } + }, + "relations": [] + }, + { + "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 3, + "factory_name": "DONG CITY TESTING NEW AUDITT", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Reviewed", + "audit_guid": "Final_test_scan", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "10.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + } + ] + }, + "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + } + }, + "relations": [] + } + ] +} \ No newline at end of file diff --git a/importers/use_cases/otjson_1.2/sort1.json b/importers/use_cases/otjson_1.2/sort1.json new file mode 100644 index 0000000000..e31ab7a760 --- /dev/null +++ b/importers/use_cases/otjson_1.2/sort1.json @@ -0,0 +1,202 @@ +{ + "@id": "", + "@type": "Dataset", + "datasetHeader": { + "OTJSONVersion": "1.2", + "datasetCreationTimestamp": "2020-06-29T11:01:09.035Z", + "datasetTitle": "", + "datasetDescription": "", + "datasetTags": [], + "relatedDatasets": [], + "validationSchemas": { + "erc725-main": { + "schemaType": "ethereum-725", + "networkId": "ganache" + }, + "merkleRoot": { + "schemaType": "merkle-root", + "networkId": "ganache", + "hubContractAddress": "0x0987197628Bb06133B6FA2409eb4cF9FCaFe8d3a" + } + }, + "dataIntegrity": { + "proofs": [ + { + "proofValue": "", + "proofType": "merkleRootHash", + "validationSchema": "/schemas/merkleRoot" + } + ] + }, + "dataCreator": { + "identifiers": [ + { + "identifierValue": "0x2Fa6d32E314AAB43a58999D6f5532A15418Da153", + "identifierType": "ERC725", + "validationSchema": "/schemas/erc725-main" + } + ] + } + }, + "@graph": [ + { + "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 3, + "factory_name": "DONG CITY TESTING NEW AUDITT", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Reviewed", + "audit_guid": "Final_test_scan", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "10.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + } + ] + }, + "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + } + }, + "relations": [] + }, + { + "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 1, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pendind Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "version": 2, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pending Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Pendind Review", + "version": 1, + "audit_status_update_time": "2020-06-15 10:28:08" + }, + { + "audit_status": "Audit Pending Review", + "version": 2, + "audit_status_update_time": "2020-06-15 10:32:37" + } + ] + }, + "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + } + }, + "relations": [] + } + ] +} \ No newline at end of file diff --git a/importers/use_cases/otjson_1.2/sort2.json b/importers/use_cases/otjson_1.2/sort2.json new file mode 100644 index 0000000000..f5d3f182d6 --- /dev/null +++ b/importers/use_cases/otjson_1.2/sort2.json @@ -0,0 +1,202 @@ +{ + "@id": "", + "@type": "Dataset", + "datasetHeader": { + "OTJSONVersion": "1.2", + "datasetCreationTimestamp": "2020-06-29T11:01:09.035Z", + "datasetTitle": "", + "datasetDescription": "", + "datasetTags": [], + "relatedDatasets": [], + "validationSchemas": { + "erc725-main": { + "schemaType": "ethereum-725", + "networkId": "ganache" + }, + "merkleRoot": { + "schemaType": "merkle-root", + "networkId": "ganache", + "hubContractAddress": "0x0987197628Bb06133B6FA2409eb4cF9FCaFe8d3a" + } + }, + "dataIntegrity": { + "proofs": [ + { + "proofValue": "", + "proofType": "merkleRootHash", + "validationSchema": "/schemas/merkleRoot" + } + ] + }, + "dataCreator": { + "identifiers": [ + { + "identifierValue": "0x2Fa6d32E314AAB43a58999D6f5532A15418Da153", + "identifierType": "ERC725", + "validationSchema": "/schemas/erc725-main" + } + ] + } + }, + "@graph": [ + { + "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 1, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pendind Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "version": 2, + "factory_name": "TESTING 123", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Pending Review", + "audit_guid": "testing_123", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "110.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Pendind Review", + "version": 1, + "audit_status_update_time": "2020-06-15 10:28:08" + }, + { + "audit_status": "Audit Pending Review", + "version": 2, + "audit_status_update_time": "2020-06-15 10:32:37" + } + ] + }, + "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + } + }, + "relations": [] + }, + { + "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", + "@type": "otObject", + "identifiers": [ + { + "@type": "id", + "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + }, + { + "@type": "id", + "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + } + ], + "properties": { + "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", + "objectType": "vocabularyElement", + "permissioned_data": { + "data": { + "audits_data": [ + { + "version": 3, + "factory_name": "DONG CITY TESTING NEW AUDITT", + "scanid": "CN563H8A91DO", + "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", + "country": "Austria", + "city": "Vienna", + "lat": "22.9", + "lon": "113.67", + "commodity_types": null, + "years_in_business": 23, + "number_of_employees": 120, + "importer_of_record_number": null, + "mutual_certificates": "Yes", + "audit_name": "2020 SCAN Testing Audit", + "audit_date": "2020-03-12 09:27:53", + "audit_status": "Audit Reviewed", + "audit_guid": "Final_test_scan", + "audit_name_guid": "testingAna123", + "compliance_score_overall": "10.00", + "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", + "risk_index": "High", + "auditor_company": "OriginTrail", + "auditor_training_certificate": null, + "geographic_risk_overview": "[]", + "CTPAT_participation": null, + "report_number": "EA-2020-03-0036", + "is_revoked": null + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + }, + { + "audit_status": "Audit Reviewed", + "version": 3, + "audit_status_update_time": "2020-06-15 09:10:36" + } + ] + }, + "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + } + }, + "relations": [] + } + ] +} \ No newline at end of file diff --git a/modules/ImportUtilities.js b/modules/ImportUtilities.js index 5ebed89ee2..ded91c561d 100644 --- a/modules/ImportUtilities.js +++ b/modules/ImportUtilities.js @@ -140,13 +140,8 @@ class ImportUtilities { } static prepareDataset(originalDocument, config, web3) { - let document = OtJsonUtilities.prepareDatasetForNewImport(originalDocument); - if (!document) { - document = originalDocument; - } - const datasetHeader = document.datasetHeader ? document.datasetHeader : {}; - ImportUtilities.calculateGraphPermissionedDataHashes(document['@graph']); - const id = ImportUtilities.calculateGraphPublicHash(document); + const datasetHeader = originalDocument.datasetHeader ? originalDocument.datasetHeader : {}; + ImportUtilities.calculateGraphPermissionedDataHashes(originalDocument['@graph']); const header = ImportUtilities.createDatasetHeader( config, null, @@ -154,18 +149,25 @@ class ImportUtilities { datasetHeader.datasetTitle, datasetHeader.datasetDescription, datasetHeader.OTJSONVersion, + datasetHeader.datasetCreationTimestamp, ); const dataset = { - '@id': id, + '@id': '', '@type': 'Dataset', datasetHeader: header, - '@graph': document['@graph'], + '@graph': originalDocument['@graph'], }; - const rootHash = ImportUtilities.calculateDatasetRootHash(dataset); - dataset.datasetHeader.dataIntegrity.proofs[0].proofValue = rootHash; + let document = OtJsonUtilities.prepareDatasetForNewImport(dataset); + if (!document) { + document = dataset; + } + document['@id'] = ImportUtilities.calculateGraphPublicHash(document); + + const rootHash = ImportUtilities.calculateDatasetRootHash(document); + document.datasetHeader.dataIntegrity.proofs[0].proofValue = rootHash; - const signed = ImportUtilities.signDataset(dataset, config, web3); + const signed = ImportUtilities.signDataset(document, config, web3); return signed; } @@ -674,10 +676,10 @@ class ImportUtilities { * Fill in dataset header * @private */ - static createDatasetHeader(config, transpilationInfo = null, datasetTags = [], datasetTitle = '', datasetDescription = '', OTJSONVersion = '1.1') { + static createDatasetHeader(config, transpilationInfo = null, datasetTags = [], datasetTitle = '', datasetDescription = '', OTJSONVersion = '1.1', datasetCreationTimestamp = new Date().toISOString()) { const header = { OTJSONVersion, - datasetCreationTimestamp: new Date().toISOString(), + datasetCreationTimestamp, datasetTitle, datasetDescription, datasetTags, diff --git a/modules/OtJsonUtilities.js b/modules/OtJsonUtilities.js index 634bcba0ab..997f91de23 100644 --- a/modules/OtJsonUtilities.js +++ b/modules/OtJsonUtilities.js @@ -47,7 +47,7 @@ class OtJsonUtilities { '@id': dataset['@id'], '@type': dataset['@type'], datasetHeader: dataset.datasetHeader, - '@graph': JSON.parse(Utilities.sortObjectRecursively(dataset['@graph'])), + '@graph': JSON.parse(Utilities.sortedStringify(dataset['@graph'], false)), signature: dataset.signature, }; } @@ -71,6 +71,7 @@ class OtJsonUtilities { datasetCopy['@graph'] = JSON.parse(Utilities.sortedStringify(datasetCopy['@graph'], true)); return datasetCopy; case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -96,6 +97,7 @@ class OtJsonUtilities { OtJsonUtilities.sortGraphRelationsAndIdentifiers(datasetCopy['@graph']); return datasetCopy; case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -122,9 +124,12 @@ class OtJsonUtilities { case '1.1': datasetCopy = Utilities.copyObject(dataset); if (datasetCopy.datasetHeader.transpilationInfo) { - return JSON.parse(Utilities.sortObjectRecursively(datasetCopy)); + return JSON.parse(Utilities.sortedStringify(datasetCopy, false)); } return this._repackDataset(datasetCopy); + case '1.2': + datasetCopy = Utilities.copyObject(dataset); + return JSON.parse(Utilities.sortObjectRecursively(datasetCopy)); default: throw new Error('Unsupported ot-json version!'); } @@ -148,6 +153,7 @@ class OtJsonUtilities { OtJsonUtilities.sortGraphRelationsAndIdentifiers(datasetCopy['@graph']); return JSON.parse(Utilities.sortedStringify(datasetCopy, false)); case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -172,6 +178,7 @@ class OtJsonUtilities { datasetCopy['@graph'] = JSON.parse(Utilities.sortedStringify(datasetCopy['@graph'], true)); return datasetCopy; case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -195,6 +202,7 @@ class OtJsonUtilities { OtJsonUtilities.sortGraphRelationsAndIdentifiers(datasetCopy['@graph']); return datasetCopy; case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -219,6 +227,7 @@ class OtJsonUtilities { OtJsonUtilities.sortGraphRelationsAndIdentifiers(datasetCopy['@graph']); return JSON.parse(Utilities.sortedStringify(datasetCopy, false)); case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -243,6 +252,7 @@ class OtJsonUtilities { OtJsonUtilities.sortGraphRelationsAndIdentifiers(datasetCopy['@graph']); return JSON.parse(Utilities.sortedStringify(datasetCopy, false)); case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -265,6 +275,12 @@ class OtJsonUtilities { case '1.0': return undefined; case '1.1': + datasetCopy = Utilities.copyObject(dataset); + if (datasetCopy.datasetHeader.transpilationInfo) { + return JSON.parse(Utilities.sortedStringify(datasetCopy, false)); + } + return this._repackDataset(datasetCopy); + case '1.2': datasetCopy = Utilities.copyObject(dataset); return JSON.parse(Utilities.sortObjectRecursively(datasetCopy)); default: @@ -290,6 +306,7 @@ class OtJsonUtilities { OtJsonUtilities.sortGraphRelationsAndIdentifiers(datasetCopy['@graph']); return datasetCopy; case '1.1': + case '1.2': return undefined; default: throw new Error('Unsupported ot-json version!'); @@ -314,9 +331,12 @@ class OtJsonUtilities { case '1.1': datasetCopy = Utilities.copyObject(dataset); if (datasetCopy.datasetHeader.transpilationInfo) { - return JSON.parse(Utilities.sortObjectRecursively(datasetCopy)); + return JSON.parse(Utilities.sortedStringify(datasetCopy, false)); } return this._repackDataset(datasetCopy); + case '1.2': + datasetCopy = Utilities.copyObject(dataset); + return JSON.parse(Utilities.sortObjectRecursively(datasetCopy)); default: throw new Error('Unsupported ot-json version!'); } @@ -340,9 +360,12 @@ class OtJsonUtilities { case '1.1': datasetCopy = Utilities.copyObject(dataset); if (datasetCopy.datasetHeader.transpilationInfo) { - return JSON.parse(Utilities.sortObjectRecursively(datasetCopy)); + return JSON.parse(Utilities.sortedStringify(datasetCopy, false)); } return this._repackDataset(datasetCopy); + case '1.2': + datasetCopy = Utilities.copyObject(dataset); + return JSON.parse(Utilities.sortObjectRecursively(datasetCopy)); default: throw new Error('Unsupported ot-json version!'); } diff --git a/modules/Utilities.js b/modules/Utilities.js index ff819c03bd..50be2fea2a 100644 --- a/modules/Utilities.js +++ b/modules/Utilities.js @@ -84,19 +84,22 @@ class Utilities { for (const key of Object.keys(obj)) { if (Array.isArray(obj)) { if (obj[key] != null && typeof obj[key] === 'object') { - stringified.push(this.sortedStringify(obj[key], inProperties)); + stringified.push(this.sortObjectRecursively(obj[key], inProperties)); } else { // Added for better performance by avoiding the last level of recursion // because the last level only returns JSON.stringify of the key - stringified.push(JSON.stringify(obj[key])); + stringified.push(JSON.stringify( + obj[key], + (k, v) => (v === undefined ? null : v), + )); } } else if (obj[key] != null && typeof obj[key] === 'object') { if (key === 'properties') { inProperties = true; } - stringified.push(`"${key}":${this.sortedStringify(obj[key], inProperties)}`); + stringified.push(`"${key}":${this.sortObjectRecursively(obj[key], inProperties)}`); } else { // Added for better performance by avoiding the last level of recursion // because the last level only returns JSON.stringify of the key - stringified.push(`"${key}":${JSON.stringify(obj[key])}`); + stringified.push(`"${key}":${JSON.stringify(obj[key], (k, v) => (v === undefined ? null : v))}`); } } @@ -112,7 +115,7 @@ class Utilities { // Return result in the format of an object return `{${stringified.join(',')}}`; } - return JSON.stringify(obj); + return JSON.stringify(obj, (k, v) => (v === undefined ? null : v)); } /** diff --git a/modules/service/import-service.js b/modules/service/import-service.js index b97f52cf9a..c17adb761d 100644 --- a/modules/service/import-service.js +++ b/modules/service/import-service.js @@ -788,7 +788,7 @@ class ImportService { // TODO: Prepare support for multiple versions const { OTJSONVersion } = datasetHeader; - if (!['1.0', '1.1'].includes(OTJSONVersion)) { + if (!['1.0', '1.1', '1.2'].includes(OTJSONVersion)) { throw Error('[Validation Error] Unsupported OT-JSON version.'); } diff --git a/test/bdd/features/importer.feature b/test/bdd/features/importer.feature index 3df984a599..bed0fa84ca 100644 --- a/test/bdd/features/importer.feature +++ b/test/bdd/features/importer.feature @@ -107,4 +107,45 @@ Feature: Test basic importer features And I use 4th node as DC And DC gets issuer id for element "1234567890000000015" And I use 1st node as DC - Then DC should be the issuer for the selected element \ No newline at end of file + Then DC should be the issuer for the selected element + + @second + Scenario: Check that two OT-JSON 1.1 datasets with different order have different hashes + Given the replication difficulty is 0 + And I setup 4 node + And I start the nodes + And I use 1st node as DC + And DC imports "importers/use_cases/otjson_1.1/sort1.json" as GRAPH + And DC waits for import to finish + And DC initiates the replication for last imported dataset + And I wait for replications to finish + And I use 3rd node as DV + When DV exports the last imported dataset as OT-JSON + And DV waits for export to finish + And I use 2st node as DC + And DC imports "importers/use_cases/otjson_1.1/sort2.json" as GRAPH + And DC waits for import to finish + And DC initiates the replication for last imported dataset + And I wait for replications to finish + When DV exports the last imported dataset as OT-JSON + And DV waits for export to finish + Then the last two exported datasets should not have the same hashes + + @third + Scenario: Check that two OT-JSON 1.2 datasets with different order have the same hashes + Given the replication difficulty is 0 + And I setup 4 node + And I start the nodes + And I use 1st node as DC + And DC imports "importers/use_cases/otjson_1.2/sort1.json" as GRAPH + And DC waits for import to finish + And DC initiates the replication for last imported dataset + And I wait for replications to finish + And I use 1st node as DV + When DV exports the last imported dataset as OT-JSON + And DV waits for export to finish + And DC imports "importers/use_cases/otjson_1.2/sort2.json" as GRAPH + And DC waits for import to finish + When DV exports the last imported dataset as OT-JSON + And DV waits for export to finish + Then the last two exported datasets should have the same hashes \ No newline at end of file diff --git a/test/bdd/steps/network.js b/test/bdd/steps/network.js index 03e76e1c9e..ddd0fb9b21 100644 --- a/test/bdd/steps/network.js +++ b/test/bdd/steps/network.js @@ -14,6 +14,7 @@ const path = require('path'); const OtNode = require('./lib/otnode'); const ImportUtilities = require('../../../modules/ImportUtilities'); +const Utilities = require('../../../modules/Utilities'); const LocalBlockchain = require('./lib/local-blockchain'); const httpApiHelper = require('./lib/http-api-helper'); const utilities = require('./lib/utilities'); @@ -441,6 +442,79 @@ Then(/^the last root hash should be the same as one manually calculated$/, async .to.be.equal(calculatedDataSetId); }); +Then(/^the last two exported datasets ([should|should not]+) have the same hashes$/, async function (condition) { + this.logger.log('The last root hash should be the same as one manually calculated$'); + expect(!!this.state.dc, 'DC node not defined. Use other step to define it.').to.be.equal(true); + expect(this.state.nodes.length, 'No started nodes').to.be.greaterThan(0); + expect(this.state.bootstraps.length, 'No bootstrap nodes').to.be.greaterThan(0); + expect(!!this.state.lastExport, 'Last export didn\'t happen. Use other step to do it.').to.be.equal(true); + expect(!!this.state.secondLastExport, 'Second last export didn\'t happen. Use other step to do it.').to.be.equal(true); + expect(!!this.state.lastImport, 'Last import didn\'t happen. Use other step to do it.').to.be.equal(true); + expect(!!this.state.secondLastImport, 'Second last import didn\'t happen. Use other step to do it.').to.be.equal(true); + + const dataset1 = JSON.parse(this.state.secondLastExport.data.formatted_dataset); + const dataset2 = JSON.parse(this.state.lastExport.data.formatted_dataset); + const dc1 = this.state.nodes[0]; + let dc2; + if (condition.includes('not')) { + dc2 = this.state.nodes[1]; + } else { + dc2 = this.state.nodes[0]; + } + + // check dataset_id + const calculatedDatasetId1 = ImportUtilities.calculateGraphPublicHash(dataset1); + expect(this.state.secondLastImport.data.dataset_id, 'Dataset from API endpoint and manually calculated should match') + .to.be.equal(calculatedDatasetId1); + + // check signature + const calcuatedDatasetSignature1 = ImportUtilities.extractDatasetSigner(dataset1, new Web3()); + expect(Utilities.normalizeHex(calcuatedDatasetSignature1), 'Dataset from API endpoint and manually calculated should match') + .to.be.equal(Utilities.normalizeHex(dc1.options.nodeConfiguration.node_wallet)); + + // check root_hash + const calculatedDatasetRootHash1 = ImportUtilities.calculateDatasetRootHash(dataset1); + expect(calculatedDatasetRootHash1, 'Dataset from API endpoint and manually calculated should match') + .to.be.equal(this.state.secondLastExport.data.root_hash); + + // check dataset_id + const calculatedDatasetId2 = ImportUtilities.calculateGraphPublicHash(dataset2); + expect(this.state.lastImport.data.dataset_id, 'Dataset from API endpoint and manually calculated should match') + .to.be.equal(calculatedDatasetId2); + + // check signature + const calcuatedDatasetSignature2 = ImportUtilities.extractDatasetSigner(dataset2, new Web3()); + expect(Utilities.normalizeHex(calcuatedDatasetSignature2), 'Dataset from API endpoint and manually calculated should match') + .to.be.equal(Utilities.normalizeHex(dc2.options.nodeConfiguration.node_wallet)); + + // check root_hash + const calculatedDatasetRootHash2 = ImportUtilities.calculateDatasetRootHash(dataset2); + expect(calculatedDatasetRootHash2, 'Dataset from API endpoint and manually calculated should match') + .to.be.equal(this.state.lastExport.data.root_hash); + + if (condition.includes('not')) { + expect(calculatedDatasetId1).to.be.not.equal(calculatedDatasetId2); + expect(calcuatedDatasetSignature1).to.be.not.equal(calcuatedDatasetSignature2); + expect(calculatedDatasetRootHash1).to.be.not.equal(calculatedDatasetRootHash2); + } else { + expect(calculatedDatasetId1).to.be.equal(calculatedDatasetId2); + expect(calcuatedDatasetSignature1).to.be.equal(calcuatedDatasetSignature2); + expect(calculatedDatasetRootHash1).to.be.equal(calculatedDatasetRootHash2); + } +}); + +Then(/^the last two datasets should have the same hashes$/, async function () { + this.logger.log('The last root hash should be the same as one manually calculated$'); + expect(!!this.state.dc, 'DC node not defined. Use other step to define it.').to.be.equal(true); + expect(this.state.nodes.length, 'No started nodes').to.be.greaterThan(0); + expect(this.state.bootstraps.length, 'No bootstrap nodes').to.be.greaterThan(0); + expect(!!this.state.lastImport, 'Last import didn\'t happen. Use other step to do it.').to.be.equal(true); + expect(!!this.state.secondLastImport, 'Last import didn\'t happen. Use other step to do it.').to.be.equal(true); + + expect(this.state.lastImport.data.dataset_id, 'Fingerprint from API endpoint and manually calculated should match') + .to.be.equal(this.state.secondLastImport.data.dataset_id); +}); + Given(/^I wait for replication[s] to finish$/, { timeout: 1200000 }, function () { this.logger.log('I wait for replication to finish'); expect(!!this.state.dc, 'DC node not defined. Use other step to define it.').to.be.equal(true); From 198f7cea38f36f22bc113fc01f1ae9a83fc9a89c Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 2 Jul 2020 13:32:43 +0200 Subject: [PATCH 5/9] added support for otjson version 1.2 --- importers/use_cases/otjson_1.1/sort1.json | 221 +++++++---------- importers/use_cases/otjson_1.1/sort2.json | 221 +++++++---------- importers/use_cases/otjson_1.2/sort1.json | 223 +++++++----------- importers/use_cases/otjson_1.2/sort2.json | 223 +++++++----------- modules/OtJsonUtilities.js | 23 +- modules/Utilities.js | 7 +- .../command/dh/dh-offer-finalized-command.js | 6 +- modules/service/dh-service.js | 2 +- test/bdd/features/importer.feature | 2 +- test/bdd/steps/network.js | 2 + 10 files changed, 387 insertions(+), 543 deletions(-) diff --git a/importers/use_cases/otjson_1.1/sort1.json b/importers/use_cases/otjson_1.1/sort1.json index f983e04de8..5a862b8ccb 100644 --- a/importers/use_cases/otjson_1.1/sort1.json +++ b/importers/use_cases/otjson_1.1/sort1.json @@ -40,163 +40,118 @@ }, "@graph": [ { - "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", - "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@type": "kind", + "@value": "test_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@value": "test_object_02" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 3, - "factory_name": "DONG CITY TESTING NEW AUDITT", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Reviewed", - "audit_guid": "Final_test_scan", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "10.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - } - ] + "purpose": "Testing object number 2", + "array": [ + "second_element", "first_element" + ] + }, + "@id": "test_object_02", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" }, - "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" + } + ] + }, + { + "identifiers": [ + { + "@type": "kind", + "@value": "test_object" + }, + { + "@type": "id", + "@value": "test_object_01" } + ], + "properties": { + "purpose": "Testing object number 1", + "array": [ + "second_element", "first_element" + ] }, - "relations": [] + "@id": "test_object_01", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" + }, + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" + } + ] }, { - "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", + "@id": "master_object_01", "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@type": "kind", + "@value": "master_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@value": "master_object_01" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 1, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pendind Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "version": 2, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pending Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Pendind Review", - "version": 1, - "audit_status_update_time": "2020-06-15 10:28:08" - }, - { - "audit_status": "Audit Pending Review", - "version": 2, - "audit_status_update_time": "2020-06-15 10:32:37" - } - ] + "purpose": "Master object number 1", + "array": [ + "second_element", "first_element" + ] + }, + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" + }, + "relationType": "IS_CHILD" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" }, - "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + "relationType": "IS_CHILD" } - }, - "relations": [] + ] } ] } \ No newline at end of file diff --git a/importers/use_cases/otjson_1.1/sort2.json b/importers/use_cases/otjson_1.1/sort2.json index f00ba6f876..b1e15d9951 100644 --- a/importers/use_cases/otjson_1.1/sort2.json +++ b/importers/use_cases/otjson_1.1/sort2.json @@ -40,163 +40,118 @@ }, "@graph": [ { - "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", - "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@type": "kind", + "@value": "test_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@value": "test_object_02" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 1, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pendind Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "version": 2, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pending Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Pendind Review", - "version": 1, - "audit_status_update_time": "2020-06-15 10:28:08" - }, - { - "audit_status": "Audit Pending Review", - "version": 2, - "audit_status_update_time": "2020-06-15 10:32:37" - } - ] + "purpose": "Testing object number 2", + "array": [ + "second_element", "first_element" + ] + }, + "@id": "test_object_02", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" }, - "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" } - }, - "relations": [] + ] }, { - "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", + "@id": "master_object_01", "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@type": "kind", + "@value": "master_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@value": "master_object_01" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 3, - "factory_name": "DONG CITY TESTING NEW AUDITT", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Reviewed", - "audit_guid": "Final_test_scan", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "10.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - } - ] + "purpose": "Master object number 1", + "array": [ + "second_element", "first_element" + ] + }, + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" }, - "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + "relationType": "IS_CHILD" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" + }, + "relationType": "IS_CHILD" } + ] + }, + { + "identifiers": [ + { + "@type": "kind", + "@value": "test_object" + }, + { + "@type": "id", + "@value": "test_object_01" + } + ], + "properties": { + "purpose": "Testing object number 1", + "array": [ + "second_element", "first_element" + ] }, - "relations": [] + "@id": "test_object_01", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" + }, + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" + } + ] } ] } \ No newline at end of file diff --git a/importers/use_cases/otjson_1.2/sort1.json b/importers/use_cases/otjson_1.2/sort1.json index e31ab7a760..ea479a441e 100644 --- a/importers/use_cases/otjson_1.2/sort1.json +++ b/importers/use_cases/otjson_1.2/sort1.json @@ -38,165 +38,120 @@ ] } }, - "@graph": [ + "@graph": [ { - "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", - "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@type": "kind", + "@value": "test_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@value": "test_object_02" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 3, - "factory_name": "DONG CITY TESTING NEW AUDITT", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Reviewed", - "audit_guid": "Final_test_scan", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "10.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - } - ] + "purpose": "Testing object number 2", + "array": [ + "second_element", "first_element" + ] + }, + "@id": "test_object_02", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" }, - "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" + } + ] + }, + { + "identifiers": [ + { + "@type": "kind", + "@value": "test_object" + }, + { + "@type": "id", + "@value": "test_object_01" } + ], + "properties": { + "purpose": "Testing object number 1", + "array": [ + "second_element", "first_element" + ] }, - "relations": [] + "@id": "test_object_01", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" + }, + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" + } + ] }, { - "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", + "@id": "master_object_01", "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@type": "kind", + "@value": "master_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@value": "master_object_01" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 1, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pendind Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "version": 2, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pending Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Pendind Review", - "version": 1, - "audit_status_update_time": "2020-06-15 10:28:08" - }, - { - "audit_status": "Audit Pending Review", - "version": 2, - "audit_status_update_time": "2020-06-15 10:32:37" - } - ] + "purpose": "Master object number 1", + "array": [ + "second_element", "first_element" + ] + }, + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" + }, + "relationType": "IS_CHILD" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" }, - "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + "relationType": "IS_CHILD" } - }, - "relations": [] + ] } ] } \ No newline at end of file diff --git a/importers/use_cases/otjson_1.2/sort2.json b/importers/use_cases/otjson_1.2/sort2.json index f5d3f182d6..92a63ee129 100644 --- a/importers/use_cases/otjson_1.2/sort2.json +++ b/importers/use_cases/otjson_1.2/sort2.json @@ -40,163 +40,118 @@ }, "@graph": [ { - "@id": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62", - "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@type": "kind", + "@value": "test_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#6a61c1fdb54678049933c5ec756a17e38e0b735052f9727619c7d9c51f3a4d62" + "@value": "test_object_02" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 1, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pendind Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "version": 2, - "factory_name": "TESTING 123", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Pending Review", - "audit_guid": "testing_123", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "110.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Pendind Review", - "version": 1, - "audit_status_update_time": "2020-06-15 10:28:08" - }, - { - "audit_status": "Audit Pending Review", - "version": 2, - "audit_status_update_time": "2020-06-15 10:32:37" - } - ] + "purpose": "Testing object number 2", + "array": [ + "second_element", "first_element" + ] + }, + "@id": "test_object_02", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" }, - "permissioned_data_hash": "0xebbfc4be47497e0a401bc2b69cdba178f94b16f1d2e94ce52e0b63a4470bee3d" + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" } - }, - "relations": [] + ] }, { - "@id": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732", + "@id": "master_object_01", "@type": "otObject", "identifiers": [ { - "@type": "id", - "@value": "c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@type": "kind", + "@value": "master_object" }, { "@type": "id", - "@value": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A#c77571a4eb41dd5b2a794b49a1c2d965083d6567de05cb2fb8893d8e9e5eb732" + "@value": "master_object_01" } ], "properties": { - "issuer": "did:ethr:0xf463132Fb992B086BF1C862b5cbC643DD5AF5F2A", - "objectType": "vocabularyElement", - "permissioned_data": { - "data": { - "audits_data": [ - { - "version": 3, - "factory_name": "DONG CITY TESTING NEW AUDITT", - "scanid": "CN563H8A91DO", - "linkedscanmembers": "[{\"name\": \"Ana\"}, {\"name\": \"SCAN1002\"}, {\"name\": \"SCAN1004\"}, {\"name\": \"SCAN 1111\"}, {\"name\": \"SCAN 123345\"}, {\"name\": \"SCAN 123144\"}, {\"name\": null}]", - "country": "Austria", - "city": "Vienna", - "lat": "22.9", - "lon": "113.67", - "commodity_types": null, - "years_in_business": 23, - "number_of_employees": 120, - "importer_of_record_number": null, - "mutual_certificates": "Yes", - "audit_name": "2020 SCAN Testing Audit", - "audit_date": "2020-03-12 09:27:53", - "audit_status": "Audit Reviewed", - "audit_guid": "Final_test_scan", - "audit_name_guid": "testingAna123", - "compliance_score_overall": "10.00", - "compliance_score_categories": "[{\"score\": \"45\", \"category\": \"Procedural Security\", \"category_guid\": \"3D7F4BD2-D66B-480D-8559-712F054325063\"}, {\"score\": \"66\", \"category\": \"Physical Security\", \"category_guid\": \"7FAD0954-4BE2-4A6D-9DF6-AF5D895723FB4\"}, {\"score\": \"54\", \"category\": \"Security Training & Threat Awareness\", \"category_guid\": \"9517FEE6-F5EB-4BFF-A524-DBBD6BCD7E345\"}, {\"score\": \"78\", \"category\": \"Conveyances and Instruments of International Traffic\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA9\"}, {\"score\": \"90\", \"category\": \"Seal Security\", \"category_guid\": \"5B97E7CF-D0A1-42D4-8596-8935770089BA11\"}]", - "risk_index": "High", - "auditor_company": "OriginTrail", - "auditor_training_certificate": null, - "geographic_risk_overview": "[]", - "CTPAT_participation": null, - "report_number": "EA-2020-03-0036", - "is_revoked": null - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - }, - { - "audit_status": "Audit Reviewed", - "version": 3, - "audit_status_update_time": "2020-06-15 09:10:36" - } - ] + "purpose": "Master object number 1", + "array": [ + "second_element", "first_element" + ] + }, + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" }, - "permissioned_data_hash": "0xea03dacd76a109fd9a6d7fdb709c3d46afed7f9bcf839875db9bb40faf3af261" + "relationType": "IS_CHILD" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_01" + }, + "relationType": "IS_CHILD" } + ] + }, + { + "identifiers": [ + { + "@type": "kind", + "@value": "test_object" + }, + { + "@type": "id", + "@value": "test_object_01" + } + ], + "properties": { + "purpose": "Testing object number 1", + "array": [ + "second_element", "first_element" + ] }, - "relations": [] + "@id": "test_object_01", + "@type": "otObject", + "relations": [ + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "test_object_02" + }, + "relationType": "IS_SIBLING" + }, + { + "@type": "otRelation", + "direction": "direct", + "linkedObject": { + "@id": "master_object_01" + }, + "relationType": "IS_PARENT" + } + ] } ] -} \ No newline at end of file +} \ No newline at end of file diff --git a/modules/OtJsonUtilities.js b/modules/OtJsonUtilities.js index 997f91de23..9a6ac4ec41 100644 --- a/modules/OtJsonUtilities.js +++ b/modules/OtJsonUtilities.js @@ -43,11 +43,30 @@ class OtJsonUtilities { } static _repackDataset(dataset) { + const config = {}; + config.blockchain = {}; + config.blockchain.network_id = dataset.datasetHeader.validationSchemas.merkleRoot.networkId; + config.blockchain.hub_contract_address = + dataset.datasetHeader.validationSchemas.merkleRoot.hubContractAddress; + config.erc725Identity = dataset.datasetHeader.dataCreator.identifiers[0].identifierValue; + // eslint-disable-next-line global-require + const header = require('./ImportUtilities').createDatasetHeader( + config, null, + dataset.datasetHeader.datasetTags, + dataset.datasetHeader.datasetTitle, + dataset.datasetHeader.datasetDescription, + dataset.datasetHeader.OTJSONVersion, + ); + header.datasetCreationTimestamp = dataset.datasetHeader.datasetCreationTimestamp; + header.dataIntegrity.proofs[0].proofValue = + dataset.datasetHeader.dataIntegrity.proofs[0].proofValue; + + const graph = JSON.parse(Utilities.sortedStringify(dataset['@graph'], false)); return { '@id': dataset['@id'], '@type': dataset['@type'], - datasetHeader: dataset.datasetHeader, - '@graph': JSON.parse(Utilities.sortedStringify(dataset['@graph'], false)), + datasetHeader: header, + '@graph': graph, signature: dataset.signature, }; } diff --git a/modules/Utilities.js b/modules/Utilities.js index 50be2fea2a..c195fda271 100644 --- a/modules/Utilities.js +++ b/modules/Utilities.js @@ -94,8 +94,11 @@ class Utilities { )); } } else if (obj[key] != null && typeof obj[key] === 'object') { - if (key === 'properties') { inProperties = true; } - stringified.push(`"${key}":${this.sortObjectRecursively(obj[key], inProperties)}`); + if (key === 'properties') { + stringified.push(`"${key}":${this.sortObjectRecursively(obj[key], true)}`); + } else { + stringified.push(`"${key}":${this.sortObjectRecursively(obj[key], inProperties)}`); + } } else { // Added for better performance by avoiding the last level of recursion // because the last level only returns JSON.stringify of the key diff --git a/modules/command/dh/dh-offer-finalized-command.js b/modules/command/dh/dh-offer-finalized-command.js index a8d745bfe4..1cd7e69c67 100644 --- a/modules/command/dh/dh-offer-finalized-command.js +++ b/modules/command/dh/dh-offer-finalized-command.js @@ -53,7 +53,7 @@ class DhOfferFinalizedCommand extends Command { await bid.save({ fields: ['status'] }); this.logger.important(`I've been chosen for offer ${offerId}.`); - await this.remoteControl.onCompletedBids(); + // await this.remoteControl.onCompletedBids(); if (this.config.disableAutoPayouts !== true) { const scheduledTime = @@ -79,7 +79,7 @@ class DhOfferFinalizedCommand extends Command { bid.status = 'NOT_CHOSEN'; await bid.save({ fields: ['status'] }); this.logger.important(`I haven't been chosen for offer ${offerId}.`); - await this.remoteControl.onCompletedBids(); + // await this.remoteControl.onCompletedBids(); return Command.empty(); } } @@ -97,7 +97,7 @@ class DhOfferFinalizedCommand extends Command { const bid = await Models.bids.findOne({ where: { offer_id: offerId } }); bid.status = 'NOT_CHOSEN'; await bid.save({ fields: ['status'] }); - await this.remoteControl.onCompletedBids(); + // await this.remoteControl.onCompletedBids(); return Command.empty(); } diff --git a/modules/service/dh-service.js b/modules/service/dh-service.js index 4a47d80cde..6d908a6fc7 100644 --- a/modules/service/dh-service.js +++ b/modules/service/dh-service.js @@ -170,7 +170,7 @@ class DHService { transactional: false, }); - await this.remoteControl.getPendingBids(); + // await this.remoteControl.getPendingBids(); } /** diff --git a/test/bdd/features/importer.feature b/test/bdd/features/importer.feature index bed0fa84ca..594e81971c 100644 --- a/test/bdd/features/importer.feature +++ b/test/bdd/features/importer.feature @@ -122,7 +122,7 @@ Feature: Test basic importer features And I use 3rd node as DV When DV exports the last imported dataset as OT-JSON And DV waits for export to finish - And I use 2st node as DC + And I use 2nd node as DC And DC imports "importers/use_cases/otjson_1.1/sort2.json" as GRAPH And DC waits for import to finish And DC initiates the replication for last imported dataset diff --git a/test/bdd/steps/network.js b/test/bdd/steps/network.js index ddd0fb9b21..edb62ab5dc 100644 --- a/test/bdd/steps/network.js +++ b/test/bdd/steps/network.js @@ -457,8 +457,10 @@ Then(/^the last two exported datasets ([should|should not]+) have the same hashe const dc1 = this.state.nodes[0]; let dc2; if (condition.includes('not')) { + // eslint-disable-next-line prefer-destructuring dc2 = this.state.nodes[1]; } else { + // eslint-disable-next-line prefer-destructuring dc2 = this.state.nodes[0]; } From 4a47a3331e5d5ffe3e9794eb0e6b51efa1b6b9fb Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Fri, 3 Jul 2020 12:52:01 +0200 Subject: [PATCH 6/9] update BDD test step --- test/bdd/features/importer.feature | 8 ++++---- test/bdd/steps/network.js | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/test/bdd/features/importer.feature b/test/bdd/features/importer.feature index 594e81971c..6a4fa4431b 100644 --- a/test/bdd/features/importer.feature +++ b/test/bdd/features/importer.feature @@ -109,7 +109,7 @@ Feature: Test basic importer features And I use 1st node as DC Then DC should be the issuer for the selected element - @second + @third Scenario: Check that two OT-JSON 1.1 datasets with different order have different hashes Given the replication difficulty is 0 And I setup 4 node @@ -129,9 +129,9 @@ Feature: Test basic importer features And I wait for replications to finish When DV exports the last imported dataset as OT-JSON And DV waits for export to finish - Then the last two exported datasets should not have the same hashes + Then the last two exported datasets from 1st and 2nd node should not have the same hashes - @third + @fourth Scenario: Check that two OT-JSON 1.2 datasets with different order have the same hashes Given the replication difficulty is 0 And I setup 4 node @@ -148,4 +148,4 @@ Feature: Test basic importer features And DC waits for import to finish When DV exports the last imported dataset as OT-JSON And DV waits for export to finish - Then the last two exported datasets should have the same hashes \ No newline at end of file + Then the last two exported datasets from 1st and 1st node should have the same hashes \ No newline at end of file diff --git a/test/bdd/steps/network.js b/test/bdd/steps/network.js index edb62ab5dc..05ac64b55c 100644 --- a/test/bdd/steps/network.js +++ b/test/bdd/steps/network.js @@ -442,7 +442,7 @@ Then(/^the last root hash should be the same as one manually calculated$/, async .to.be.equal(calculatedDataSetId); }); -Then(/^the last two exported datasets ([should|should not]+) have the same hashes$/, async function (condition) { +Then(/^the last two exported datasets from (\d+)[st|nd|rd|th]+ and (\d+)[st|nd|rd|th]+ node ([should|should not]+) have the same hashes$/, async function (nodeIndex1, nodeIndex2, condition) { this.logger.log('The last root hash should be the same as one manually calculated$'); expect(!!this.state.dc, 'DC node not defined. Use other step to define it.').to.be.equal(true); expect(this.state.nodes.length, 'No started nodes').to.be.greaterThan(0); @@ -454,15 +454,16 @@ Then(/^the last two exported datasets ([should|should not]+) have the same hashe const dataset1 = JSON.parse(this.state.secondLastExport.data.formatted_dataset); const dataset2 = JSON.parse(this.state.lastExport.data.formatted_dataset); - const dc1 = this.state.nodes[0]; - let dc2; - if (condition.includes('not')) { - // eslint-disable-next-line prefer-destructuring - dc2 = this.state.nodes[1]; - } else { - // eslint-disable-next-line prefer-destructuring - dc2 = this.state.nodes[0]; - } + const dc1 = this.state.nodes[nodeIndex1 - 1]; + const dc2 = this.state.nodes[nodeIndex2 - 1]; + // let dc2; + // if (condition.includes('not')) { + // // eslint-disable-next-line prefer-destructuring + // dc2 = this.state.nodes[1]; + // } else { + // // eslint-disable-next-line prefer-destructuring + // dc2 = this.state.nodes[0]; + // } // check dataset_id const calculatedDatasetId1 = ImportUtilities.calculateGraphPublicHash(dataset1); From 9bd5c87b5a73877b5d3c747ad86d5d22dd314c5c Mon Sep 17 00:00:00 2001 From: Uros Kukic <33048701+Kuki145@users.noreply.github.com> Date: Mon, 6 Jul 2020 12:02:04 +0200 Subject: [PATCH 7/9] Preserve handler_id for a delayed offer (#1295) * Update offer status when delayed * Handle offer commands being delayed * Update handlers on repeating commands * Cancel sending replication if replication isn't in 'started' state and minor format changes * Wait longer for an offer to be finalized * Only delete non-pending handler entries --- .../common/handler_ids_cleaner_command.js | 1 + .../command/dc/dc-offer-create-bc-command.js | 76 ++++++++++++++--- .../command/dc/dc-offer-finalize-command.js | 81 +++++++++++++++---- .../command/dc/dc-offer-finalized-command.js | 2 + modules/command/dc/dc-offer-task-command.js | 7 +- .../command/dh/dh-offer-finalized-command.js | 5 +- modules/service/dc-service.js | 8 +- modules/service/rest-api-v2.js | 9 ++- 8 files changed, 156 insertions(+), 33 deletions(-) diff --git a/modules/command/common/handler_ids_cleaner_command.js b/modules/command/common/handler_ids_cleaner_command.js index e45934beca..433eab8e06 100644 --- a/modules/command/common/handler_ids_cleaner_command.js +++ b/modules/command/common/handler_ids_cleaner_command.js @@ -22,6 +22,7 @@ class HandlerIdsCleanerCommand extends Command { await Models.handler_ids.destroy({ where: { timestamp: { [Models.Sequelize.Op.lt]: timeToBeDeleted }, + status: { [Models.Sequelize.Op.in]: ['COMPLETED', 'FAILED'] }, }, }); return Command.repeat(); diff --git a/modules/command/dc/dc-offer-create-bc-command.js b/modules/command/dc/dc-offer-create-bc-command.js index 8411f08cd9..b599eae31b 100644 --- a/modules/command/dc/dc-offer-create-bc-command.js +++ b/modules/command/dc/dc-offer-create-bc-command.js @@ -32,6 +32,7 @@ class DCOfferCreateBcCommand extends Command { tokenAmountPerHolder, dataSizeInBytes, litigationIntervalInMinutes, + handler_id, urgent, } = command.data; @@ -54,7 +55,21 @@ class DCOfferCreateBcCommand extends Command { ); } catch (error) { if (error.message.includes('Gas price higher than maximum allowed price')) { - this.logger.info('Gas price too high, delaying call for 30 minutes'); + const delay = constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS / 60 / 1000; + this.logger.info(`Gas price too high, delaying call for ${delay} minutes`); + + const handler = await Models.handler_ids.findOne({ + where: { handler_id }, + }); + const handler_data = JSON.parse(handler.data); + handler_data.status = 'DELAYED'; + handler.timestamp = Date.now(); + handler.data = JSON.stringify(handler_data); + await handler.save({ fields: ['data', 'timestamp'] }); + + const message = `Offer creation has been delayed on ${(new Date(Date.now())).toUTCString()} due to high gas price`; + await Models.offers.update({ message }, { where: { id: internalOfferId } }); + return Command.repeat(); } throw error; @@ -70,6 +85,8 @@ class DCOfferCreateBcCommand extends Command { id: internalOfferId, }); + await Models.handler_ids.update({ timestamp: Date.now() }, { where: { handler_id } }); + await this.blockchain.executePlugin('fingerprint-plugin', { dataSetId, dataRootHash, @@ -85,19 +102,57 @@ class DCOfferCreateBcCommand extends Command { * @param err */ async recover(command, err) { - const { internalOfferId, handler_id } = command.data; + return this.invalidateOffer(command, err); + } + + /** + * Execute strategy when event is too late + * @param command + */ + async expired(command) { + return this.invalidateOffer( + command, + Error('The offer creation command is too late.'), + ); + } + + async invalidateOffer(command, err) { + const { dataSetId, internalOfferId, handler_id } = command.data; + this.logger.notify(`Offer for data set ${dataSetId} has not been started. ${err.message}`); + + const errorData = { + internalOfferId, + }; + const offer = await Models.offers.findOne({ where: { id: internalOfferId } }); - offer.status = 'FAILED'; - offer.global_status = 'FAILED'; - offer.message = err.message; - await offer.save({ fields: ['status', 'message', 'global_status'] }); + if (offer) { + offer.status = 'FAILED'; + offer.global_status = 'FAILED'; + offer.message = `Offer for data set ${dataSetId} has not been started. ${err.message}`; + await offer.save({ fields: ['status', 'message', 'global_status'] }); + + errorData.tokenAmountPerHolder = offer.token_amount_per_holder; + errorData.litigationIntervalInMinutes = offer.litigation_interval_in_minutes; + errorData.datasetId = offer.data_set_id; + errorData.holdingTimeInMinutes = offer.holding_time_in_minutes; + + await this.replicationService.cleanup(offer.id); + } else { + this.logger.warn(`Offer with internal id ${internalOfferId} not found in database.`); + } + this.remoteControl.offerUpdate({ id: internalOfferId, }); - Models.handler_ids.update({ - status: 'FAILED', - }, { where: { handler_id } }); - await this.replicationService.cleanup(offer.id); + + await Models.handler_ids.update({ status: 'FAILED' }, { where: { handler_id } }); + + this.errorNotificationService.notifyError( + err, + errorData, + constants.PROCESS_NAME.offerHandling, + ); + return Command.empty(); } @@ -111,6 +166,7 @@ class DCOfferCreateBcCommand extends Command { name: 'dcOfferCreateBcCommand', delay: 0, period: constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS, + deadline_at: Date.now() + (5 * constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS), transactional: false, }; Object.assign(command, map); diff --git a/modules/command/dc/dc-offer-finalize-command.js b/modules/command/dc/dc-offer-finalize-command.js index 91abd49f24..796f0ec515 100644 --- a/modules/command/dc/dc-offer-finalize-command.js +++ b/modules/command/dc/dc-offer-finalize-command.js @@ -86,8 +86,21 @@ class DCOfferFinalizeCommand extends Command { ); } catch (error) { if (error.message.includes('Gas price higher than maximum allowed price')) { - this.logger.info('Gas price too high, delaying call for 30 minutes'); - // TODO Update status in handler in case of waiting + const delay = constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS / 60 / 1000; + this.logger.warn(`Gas price too high, delaying call for ${delay} minutes`); + + const handler = await Models.handler_ids.findOne({ + where: { handler_id }, + }); + const handler_data = JSON.parse(handler.data); + handler_data.status = 'DELAYED'; + handler.timestamp = Date.now(); + handler.data = JSON.stringify(handler_data); + await handler.save({ fields: ['data', 'timestamp'] }); + + const message = `Offer finalization has been delayed on ${(new Date(Date.now())).toUTCString()} due to high gas price`; + await Models.offers.update({ message }, { where: { offer_id: offerId } }); + return Command.repeat(); } throw error; @@ -95,6 +108,9 @@ class DCOfferFinalizeCommand extends Command { const offer = await Models.offers.findOne({ where: { offer_id: offerId } }); offer.offer_finalize_transaction_hash = result.transactionHash; await offer.save({ fields: ['offer_finalize_transaction_hash'] }); + + await Models.handler_ids.update({ timestamp: Date.now() }, { where: { handler_id } }); + return { commands: [ { @@ -162,28 +178,58 @@ class DCOfferFinalizeCommand extends Command { } } err.message = errorMessage; - this.logger.error(`Offer ${offerId} has not been finalized. ${errorMessage}`); + return this.invalidateOffer(command, err); + } + + /** + * Execute strategy when event is too late + * @param command + */ + async expired(command) { + return this.invalidateOffer( + command, + Error('The offer finalization command is too late.'), + ); + } + + async invalidateOffer(command, err) { + const { + offerId, + solution, + handler_id, + } = command.data; + this.logger.error(`Offer ${offerId} has not been finalized. ${err}`); + + const errorData = { + offerId, + }; + + const offer = await Models.offers.findOne({ where: { offer_id: offerId } }); + if (offer) { + offer.status = 'FAILED'; + offer.global_status = 'FAILED'; + offer.message = `Offer ${offerId} has not been finalized. ${err.message}`; + await offer.save({ fields: ['status', 'message', 'global_status'] }); + + errorData.tokenAmountPerHolder = offer.token_amount_per_holder; + errorData.litigationIntervalInMinutes = offer.litigation_interval_in_minutes; + errorData.datasetId = offer.data_set_id; + errorData.holdingTimeInMinutes = offer.holding_time_in_minutes; + + await this.replicationService.cleanup(offer.id); + } else { + this.logger.warn(`Offer ${offerId} not found in database.`); + } - offer.status = 'FAILED'; - offer.global_status = 'FAILED'; - offer.message = `Offer for ${offerId} has not been finalized. ${errorMessage}`; - await offer.save({ fields: ['status', 'message', 'global_status'] }); this.remoteControl.offerUpdate({ offer_id: offerId, }); - Models.handler_ids.update({ - status: 'FAILED', - }, { where: { handler_id } }); + + Models.handler_ids.update({ status: 'FAILED' }, { where: { handler_id } }); this.errorNotificationService.notifyError( err, - { - offerId: offer.offer_id, - tokenAmountPerHolder: offer.token_amount_per_holder, - litigationIntervalInMinutes: offer.litigation_interval_in_minutes, - datasetId: offer.data_set_id, - holdingTimeInMinutes: offer.holding_time_in_minutes, - }, + errorData, constants.PROCESS_NAME.offerHandling, ); @@ -200,6 +246,7 @@ class DCOfferFinalizeCommand extends Command { name: 'dcOfferFinalizeCommand', delay: 0, period: constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS, + deadline_at: Date.now() + (5 * constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS), transactional: false, }; Object.assign(command, map); diff --git a/modules/command/dc/dc-offer-finalized-command.js b/modules/command/dc/dc-offer-finalized-command.js index beb5b84615..d33894ad13 100644 --- a/modules/command/dc/dc-offer-finalized-command.js +++ b/modules/command/dc/dc-offer-finalized-command.js @@ -119,6 +119,8 @@ class DcOfferFinalizedCommand extends Command { }; } } + + await Models.handler_ids.update({ timestamp: Date.now() }, { where: { handler_id } }); return Command.repeat(); } diff --git a/modules/command/dc/dc-offer-task-command.js b/modules/command/dc/dc-offer-task-command.js index 1f67263b4a..4ed1cd59de 100644 --- a/modules/command/dc/dc-offer-task-command.js +++ b/modules/command/dc/dc-offer-task-command.js @@ -76,6 +76,8 @@ class DcOfferTaskCommand extends Command { this.logger.trace(`Offer successfully started for data set ${dataSetIdNorm}. Offer ID ${eventOfferId}. Internal offer ID ${internalOfferId}.`); return this.continueSequence(this.pack(command.data), command.sequence); } + + await Models.handler_ids.update({ timestamp: Date.now() }, { where: { handler_id } }); return Command.repeat(); } @@ -84,7 +86,10 @@ class DcOfferTaskCommand extends Command { * @param command */ async expired(command) { - return this.invalidateOffer(command); + return this.invalidateOffer( + command, + Error('The offer task event is too late.'), + ); } /** diff --git a/modules/command/dh/dh-offer-finalized-command.js b/modules/command/dh/dh-offer-finalized-command.js index a8d745bfe4..a6e505cdd0 100644 --- a/modules/command/dh/dh-offer-finalized-command.js +++ b/modules/command/dh/dh-offer-finalized-command.js @@ -1,6 +1,7 @@ const Command = require('../command'); const Utilities = require('../../Utilities'); const Models = require('../../../models/index'); +const constants = require('../../constants'); /** * Repeatable command that checks whether offer is ready or not @@ -93,7 +94,7 @@ class DhOfferFinalizedCommand extends Command { async expired(command) { const { offerId } = command.data; - this.logger.important(`I haven't been chosen for offer ${offerId}. Offer has not been finalized.`); + this.logger.important(`Offer ${offerId} has not been finalized.`); const bid = await Models.bids.findOne({ where: { offer_id: offerId } }); bid.status = 'NOT_CHOSEN'; await bid.save({ fields: ['status'] }); @@ -111,7 +112,7 @@ class DhOfferFinalizedCommand extends Command { name: 'dhOfferFinalizedCommand', delay: 0, period: 10 * 1000, - deadline_at: Date.now() + (60 * 60 * 1000), // On hour. + deadline_at: Date.now() + (6 * constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS), transactional: false, }; Object.assign(command, map); diff --git a/modules/service/dc-service.js b/modules/service/dc-service.js index 07f105585c..d7b9ed8318 100644 --- a/modules/service/dc-service.js +++ b/modules/service/dc-service.js @@ -30,6 +30,8 @@ class DCService { * @param tokenAmountPerHolder * @param dataSizeInBytes * @param litigationIntervalInMinutes + * @param handler_id + * @param urgent * @returns {Promise<*>} */ async createOffer( @@ -293,6 +295,7 @@ class DCService { const message = `Replication request for offer external ${offerId} that is not in STARTED state.`; this.logger.warn(message); await this.transport.sendResponse(response, { status: 'fail', message }); + return; } const dhReputation = await this.getReputationForDh(dhIdentity); @@ -301,9 +304,10 @@ class DCService { const message = `Replication request from holder identity ${dhIdentity} declined! Unacceptable reputation: ${dhReputation.toString()}.`; this.logger.info(message); await this.transport.sendResponse(response, { status: 'fail', message }); - } else { - await this._sendReplication(offer, wallet, identity, dhIdentity, response); + return; } + + await this._sendReplication(offer, wallet, identity, dhIdentity, response); } /** diff --git a/modules/service/rest-api-v2.js b/modules/service/rest-api-v2.js index ab347687cb..36beb040ce 100644 --- a/modules/service/rest-api-v2.js +++ b/modules/service/rest-api-v2.js @@ -590,7 +590,14 @@ class RestAPIServiceV2 { }; const offer = await Models.offers.findOne({ where: { - offer_id: handlerData.offer_id, + [Models.Sequelize.Op.or]: [ + { + offer_id: handlerData.offer_id, + }, + { + id: handlerData.offer_id, + }, + ], }, }); if (offer) { From c7e2e30a7477bd42124a0bd1bcd08c760f1e8a19 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Mon, 6 Jul 2020 16:27:56 +0200 Subject: [PATCH 8/9] version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 515710e4ea..b07ddf93da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "4.1.6", + "version": "4.1.7", "description": "OriginTrail node", "main": ".eslintrc.js", "config": { From d3fbf27cf28518df658cb96d2f10b30348a59718 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 7 Jul 2020 09:24:49 +0200 Subject: [PATCH 9/9] remove comments --- test/bdd/steps/network.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/bdd/steps/network.js b/test/bdd/steps/network.js index 05ac64b55c..f0fdedefcc 100644 --- a/test/bdd/steps/network.js +++ b/test/bdd/steps/network.js @@ -456,14 +456,6 @@ Then(/^the last two exported datasets from (\d+)[st|nd|rd|th]+ and (\d+)[st|nd|r const dataset2 = JSON.parse(this.state.lastExport.data.formatted_dataset); const dc1 = this.state.nodes[nodeIndex1 - 1]; const dc2 = this.state.nodes[nodeIndex2 - 1]; - // let dc2; - // if (condition.includes('not')) { - // // eslint-disable-next-line prefer-destructuring - // dc2 = this.state.nodes[1]; - // } else { - // // eslint-disable-next-line prefer-destructuring - // dc2 = this.state.nodes[0]; - // } // check dataset_id const calculatedDatasetId1 = ImportUtilities.calculateGraphPublicHash(dataset1);