diff --git a/importers/Transformation.xml b/importers/Transformation.xml new file mode 100644 index 0000000000..428e9bfa6b --- /dev/null +++ b/importers/Transformation.xml @@ -0,0 +1,143 @@ + + + + + + 1.0 + + + CARENGINES_PROVIDER_ID + + Mary Smith + Mary_Smith@carengines.com + + + + AIRTRANSPORT_PROVIDER_ID + + John Doe + John_Doe@airtransport.com + + + + GS1 + V1.3 + 100003 + Order + 2003-05-09T00:31:52Z + + + + BusinessProcess + Order-Sell/version2-251 + EDI-Order-Sell + + + + + + + + + + Bob Johnson + Person + + + + + + + Engine + Airplane Engine for Boing + + + Transport Pallet + Pallet + + + + + + + 8635411.000333 + 2018-03-03T00:01:54Z + 2018-04-03T00:01:54Z + + + 8635411.000333 + 2018-03-03T00:02:54Z + 2018-04-03T00:02:54Z + + + 8635411.000333 + 2018-03-03T00:03:54Z + 2018-04-03T00:03:54Z + + + 8635 + 2018-03-03T00:03:54Z + + + + + + + Physical location + Storage F12 + + + + + + + + + + + + 2015-03-15T00:00:00.000-04:00 + -04:00 + + urn:epc:id:sgtin:8635411.000333.00001 + urn:epc:id:sgtin:8635411.000333.00002 + urn:epc:id:sgtin:8635411.000333.00003 + + + + urn:epc:id:sgtin:8635411.000333.00001 + 1.0 + PCS + + + urn:epc:id:sgtin:8635411.000333.00002 + 1.0 + PCS + + + urn:epc:id:sgtin:8635411.000333.00001 + 1.0 + PCS + + + + urn:epc:id:sgtin:8635.737 + + + + urn:epc:id:sgtin:8635.737 + 1 + PCS + + + BOM12345PO987 + urn:epcglobal:cbv:bizstep:creating_class_instance + + ot:event:Transformation + Packaging + + + + + + diff --git a/importers/Transport Ownership Observation.xml b/importers/Transport Ownership Observation.xml new file mode 100644 index 0000000000..adbf94e1c7 --- /dev/null +++ b/importers/Transport Ownership Observation.xml @@ -0,0 +1,251 @@ + + + + + + 1.0 + + + CARENGINES_PROVIDER_ID + + Mary Smith + Mary_Smith@carengines.com + + + + AIRTRANSPORT_PROVIDER_ID + + John Doe + John_Doe@airtransport.com + + + + GS1 + V1.3 + 100003 + Shipment + 2003-07-09T00:31:52Z + + + + BusinessProcess + Shipment/version2-251 + EDI-Shipment + + + + + + + + + + Boing AB-123 + Machine + + + + + + + Transport Pallet + Pallet + + + + + + + urn:ot:mda:product:id:#1A + 2018-03-03T00:01:54Z + 2018-04-03T00:01:54Z + + + urn:ot:mda:product:id:#1A + 2018-03-03T00:02:54Z + 2018-04-03T00:02:54Z + + + urn:ot:mda:product:id:#1A + 2018-03-03T00:03:54Z + 2018-04-03T00:03:54Z + + + urn:ot:mda:product:id:#1A + 2018-03-03T00:03:54Z + + + + + + + Physical location + Storage F12 + + urn:epc:id:sgln:6104898.16234.1 + urn:epc:id:sgln:6104898.16234.2 + urn:epc:id:sgln:6104898.16234.3 + + + + Physical location + Storage F13 + + urn:epc:id:sgln:6104898.55555.1 + urn:epc:id:sgln:6104898.55555.2 + urn:epc:id:sgln:6104898.55555.3 + + + + + + + + + + + + + 2015-04-16T00:00:00.000-04:00 + -04:00 + + urn:epc:id:sgtin:8635411.000333.00001 + + OBSERVE + urn:epcglobal:cbv:bizstep:shipping + urn:epcglobal:cbv:disp:active + + urn:ot:mda:location:Storage.F13 + + + urn:ot:mda:location:Storage.F13 + + + + + urn:epc:id:sgtin:8635411.000333.00001 + 1.0 + PCS + + + + ot:events:Transport + ot:events:Ownership + Shipment + + urn:epc:id:sgln:6104898.16234.1 + + + urn:epc:id:sgln:6104898.55555.1 + + + + + + + 2015-04-17T00:00:00.000-04:00 + -04:00 + + urn:epc:id:sgtin:8635411.000333.00001 + + OBSERVE + urn:epcglobal:cbv:bizstep:shipping + urn:epcglobal:cbv:disp:active + + urn:ot:mda:location:Storage.F13 + + + urn:ot:mda:location:Storage.F13 + + + + + urn:epc:id:sgtin:8635411.000333.00001 + 1.0 + PCS + + + + Transport + Receipt + + urn:epc:id:sgln:6104898.16234.1 + + + urn:epc:id:sgln:6104898.55555.1 + + + + + + + 2015-04-17T00:00:00.000-04:00 + -04:00 + + urn:epc:id:sgtin:8635411.000333.00001 + + OBSERVE + urn:epcglobal:cbv:bizstep:shipping + urn:epcglobal:cbv:disp:active + + urn:epc:id:sgln:6104898.55555.1 + + + urn:ot:mda:location:Storage.F13 + + + + + urn:epc:id:sgtin:8635411.000333.00001 + 1.0 + PCS + + + + ot:events:Ownership + Receipt + + urn:epc:id:sgln:6104898.16234.1 + + + urn:epc:id:sgln:6104898.55555.1 + + + + + + + 2015-04-16T00:00:00.000-04:00 + -04:00 + + urn:epc:id:sgtin:8635411.000333.00001 + + OBSERVE + urn:epcglobal:cbv:bizstep:shipping + urn:epcglobal:cbv:disp:active + + urn:epc:id:sgln:6104898.16234.3 + + + urn:ot:mda:location:Storage.F13 + + + + + urn:epc:id:sgtin:8635411.000333.00001 + 1.0 + PCS + + + + ot:events:Observation + Measurment + + 12 + 12 + + + + + + diff --git a/modules/Database/Arangojs.js b/modules/Database/Arangojs.js index 1d368627c2..6b5f6e607e 100644 --- a/modules/Database/Arangojs.js +++ b/modules/Database/Arangojs.js @@ -213,23 +213,27 @@ class ArangoJS { } /** - * Gets max vertex_key where uid is the same and has the max version + * Gets max where uid is the same and has the max version * @param uid Vertex uid * @return {Promise} */ - getVertexKeyWithMaxVersion(uid) { + getVertexWithMaxVersion(uid) { return new Promise((resolve, reject) => { const queryString = 'FOR v IN ot_vertices ' + 'FILTER v.identifiers.uid == @uid ' + 'SORT v.version DESC ' + 'LIMIT 1 ' + - 'RETURN v.vertex_key'; + 'RETURN v'; const params = { uid, }; this.runQuery(queryString, params).then((result) => { - resolve(result); + if (result.length > 0) { + resolve(result[0]); + return; + } + resolve(null); }).catch((err) => { reject(err); }); diff --git a/modules/Database/GraphStorage.js b/modules/Database/GraphStorage.js index d4b91b98f2..95a4e019b5 100644 --- a/modules/Database/GraphStorage.js +++ b/modules/Database/GraphStorage.js @@ -7,6 +7,8 @@ const log = Utilities.getLogger(); class GraphStorage { constructor(selectedDatabase) { this.selectedDatabase = selectedDatabase; + this._allowedClasses = ['Location', 'Actor', 'Product', 'Transport', + 'Transformation', 'Observation', 'Ownership']; } /** @@ -14,7 +16,7 @@ class GraphStorage { * @returns {Promise} */ connect() { - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { if (!this.selectedDatabase) { reject(Error('Unable to connect to graph database')); } else { @@ -27,6 +29,7 @@ class GraphStorage { this.selectedDatabase.host, this.selectedDatabase.port, ); + await this.__initDatabase__(); resolve(this.db); break; case 'neo4j': @@ -37,6 +40,7 @@ class GraphStorage { this.selectedDatabase.host, this.selectedDatabase.port, ); + await this.__initDatabase__(); resolve(this.db); break; default: @@ -110,12 +114,12 @@ class GraphStorage { * @param uid Vertex uid * @return {Promise} */ - getVertexKeyWithMaxVersion(uid) { + getVertexWithMaxVersion(uid) { return new Promise((resolve, reject) => { if (!this.db) { reject(Error('Not connected to graph database')); } else { - this.db.getVertexKeyWithMaxVersion(uid).then((result) => { + this.db.getVertexWithMaxVersion(uid).then((result) => { resolve(result); }).catch((err) => { reject(err); @@ -171,44 +175,6 @@ class GraphStorage { return this.db.updateDocumentImports(collectionName, document, importNumber); } - /** - * Create document collection, if collection does not exist - * @param collectionName - * @return {Promise} - */ - createCollection(collectionName) { - return new Promise((resolve, reject) => { - if (!this.db) { - reject(Error('Not connected to graph database')); - } else { - this.db.createCollection(collectionName).then((result) => { - resolve(result); - }).catch((err) => { - reject(err); - }); - } - }); - } - - /** - * Creates edge collection, if collection does not exist - * @param collectionName - * @return {Promise} - */ - createEdgeCollection(collectionName) { - return new Promise((resolve, reject) => { - if (!this.db) { - reject(Error('Not connected to graph database')); - } else { - this.db.createEdgeCollection(collectionName).then((result) => { - resolve(result); - }).catch((err) => { - reject(err); - }); - } - }); - } - /** * Get list of vertices by import ID * @param importId Import ID @@ -246,6 +212,32 @@ class GraphStorage { } }); } + + /** + * + * @param className + * @returns {Promise} + */ + async getClassId(className) { + const id = this._allowedClasses.find(element => element.toLocaleLowerCase() === + className.toLocaleLowerCase()); + return id; + } + + /** + * Initializes database with predefined collections and vertices. + * @returns {Promise} + * @private + */ + async __initDatabase__() { + await this.db.createCollection('ot_vertices'); + await this.db.createEdgeCollection('ot_edges'); + + await Promise.all(this._allowedClasses.map(className => this.db.addDocument('ot_vertices', { + _key: className, + vertex_type: 'CLASS', + }))); + } } module.exports = GraphStorage; diff --git a/modules/Database/Neo4j.js b/modules/Database/Neo4j.js index f2caf6bc62..33f6b42cf6 100644 --- a/modules/Database/Neo4j.js +++ b/modules/Database/Neo4j.js @@ -238,15 +238,18 @@ class Neo4jDB { } /** - * Gets max vertex_key where uid is the same and has the max version + * Gets max where uid is the same and has the max version * @param uid Vertex uid * @return {Promise} */ - async getVertexKeyWithMaxVersion(uid) { + async getVertexWithMaxVersion(uid) { const session = this.driver.session(); - const result = await session.run('MATCH (n)-[:CONTAINS]->(i) WHERE i.uid = $uid RETURN n._key AS v ORDER BY v DESC LIMIT 1', { uid }); + const result = await session.run('MATCH (n)-[:CONTAINS]->(i) WHERE i.uid = $uid RETURN n ORDER BY n.version DESC LIMIT 1', { uid }); session.close(); - return result.records[0]._fields[0]; + if (result.records.length > 0) { + return this._fetchVertex('_key', result.records[0]._fields[0].properties._key); + } + return null; } /** @@ -395,7 +398,20 @@ class Neo4jDB { return []; } const session = this.driver.session(); - return session.run(`MATCH(n) WHERE n._key = '${document._key}' SET n.imports = n.imports + ${importNumber} return n`); + const result = await session.run('MATCH (n) WHERE n._key = $_key RETURN n', { + _key: document._key, + }); + let { imports } = result.records[0]._fields[0].properties; + if (imports) { + imports.push(importNumber); + } else { + imports = [importNumber]; + } + await session.run('MATCH(n) WHERE n._key = $_key SET n.imports = $imports return n', { + _key: document._key, + imports, + }); + session.close(); } /** diff --git a/modules/EventHandlers.js b/modules/EventHandlers.js index e70f5b3535..5d156d1762 100644 --- a/modules/EventHandlers.js +++ b/modules/EventHandlers.js @@ -32,25 +32,40 @@ globalEmitter.on('trail', (data) => { data.response.send(500); // TODO rethink about status codes }); }); -globalEmitter.on('gs1-import-request', (data) => { - importer.importXMLgs1(data.filepath).then((response) => { - const { - data_id, - root_hash, - total_documents, - vertices, - } = response; - - Storage.connect().then(() => { - Storage.runSystemQuery('INSERT INTO data_info (data_id, root_hash, import_timestamp, total_documents) values(?, ? , ? , ?)', [data_id, root_hash, total_documents]) - .then((data_info) => { - DCService.createOffer(data_id, root_hash, total_documents, vertices); - }); - }).catch((err) => { - log.warn(err); +globalEmitter.on('gs1-import-request', async (data) => { + const response = await importer.importXMLgs1(data.filepath); + + if (response === null) { + data.response.send({ + status: 500, + message: 'Failed to parse XML.', }); - }).catch((e) => { - console.log(e); + return; + } + + const { + data_id, + root_hash, + total_documents, + vertices, + } = response; + + try { + await Storage.connect(); + await Storage.runSystemQuery('INSERT INTO data_info (data_id, root_hash, import_timestamp, total_documents) values(?, ? , ? , ?)', [data_id, root_hash, total_documents]); + await DCService.createOffer(data_id, root_hash, total_documents, vertices); + } catch (error) { + log.error(`Failed to start offer. Error ${error}.`); + data.response.send({ + status: 500, + message: 'Failed to parse XML.', + }); + return; + } + + data.response.send({ + status: 200, + message: 'Ok.', }); }); diff --git a/modules/ZK.js b/modules/ZK.js new file mode 100644 index 0000000000..8b986d8b07 --- /dev/null +++ b/modules/ZK.js @@ -0,0 +1,168 @@ +const sha3 = require('solidity-sha3').default; +const BN = require('bn.js'); +const crypto = require('crypto'); + +class ZK { + constructor() { + this.zero = new BN(0); + this.one = new BN(1); + this.p = new BN(941078291); + this.q = new BN(941072309); + this.n = this.p.mul(this.q); + this.nSquare = this.n.mul(this.n); + this.red = BN.red(this.n); + this.redSquare = BN.red(this.nSquare); + this.g = this.n.add(this.one).toRed(this.redSquare); + } + + encrypt(m, r) { + return (this.g).redPow(m).redMul(r.redPow(this.n)); + } + + generatePrime() { + let isPrime; + let pr; + + const thousand = new BN(1000); + + do { + isPrime = false; + + pr = crypto.randomBytes(8); + pr = (new BN(pr.toString('hex'))).add(thousand); + + isPrime = this.n.gcd(pr).eq(this.one) && !pr.gte(this.n); + } while (isPrime !== true); + + return pr; + } + + P(importId, eventId, inputQuantities, outputQuantities) { + const e = new BN(parseInt(sha3(importId, eventId).substring('0', '10'), 10)); + + let r = new BN(this.generatePrime()).mod(this.n); + + const a = this.encrypt(this.zero, r.toRed(this.redSquare)); + + const inputs = []; + const outputs = []; + + let R = new BN(1).toRed(this.redSquare); + let Z = this.one.toRed(this.redSquare); + + // let rs = []; + + for (const i in inputQuantities) { + const rawQuantity = inputQuantities[i].quantity; + const quantity = new BN(rawQuantity); + const { unit } = inputQuantities[i]; + + let randomness; + if (inputQuantities[i].r !== undefined) { + randomness = new BN(inputQuantities[i].r).mod(this.n).toRed(this.redSquare); + } else { + randomness = new BN(this.generatePrime()).mod(this.n).toRed(this.redSquare); + } + + const encryptedInput = this.encrypt(quantity, randomness); + // let encryptedNegInput = this.encrypt(this.n.sub(quantity), negRandomness); + + // rs.push(randomness.toNumber()) + + R = R.redMul(randomness); + Z = Z.redMul(encryptedInput); + + inputs.push({ + object: inputQuantities[i].object, + added: inputQuantities[i].added, + public: { + enc: encryptedInput, + unit, + }, + private: { + r: randomness, + quantity: rawQuantity, + unit, + }, + }); + } + + for (const i in outputQuantities) { + const rawQuantity = outputQuantities[i].quantity; + const quantity = new BN(rawQuantity); + const { unit } = outputQuantities[i]; + + let randomness; + if (outputQuantities[i].r !== undefined) { + randomness = new BN(outputQuantities[i].r).mod(this.n).toRed(this.redSquare); + } else { + randomness = new BN(this.generatePrime()).mod(this.n).toRed(this.redSquare); + } + + const encryptedOutput = this.encrypt(quantity, randomness); + const encryptedNegOutput = encryptedOutput.redInvm(); + + // rs.push(randomness.toNumber()) + + R = R.redMul(randomness.redInvm()); + Z = Z.redMul(encryptedNegOutput); + + outputs.push({ + object: outputQuantities[i].object, + added: outputQuantities[i].added, + public: { + enc: encryptedOutput.toString('hex'), + // encNeg: '0x' + encryptedNegOutput.toString('hex'), + unit, + }, + private: { + object: outputQuantities[i].object, + r: randomness, + // rp : negRandomness, + quantity: rawQuantity, + unit, + }, + }); + } + + + r = r.toRed(this.redSquare); + const zp = r.redMul(R.redPow(e)); + + const res = this.V(e, a, Z, zp); + /* + if (res == false) { + console.log(R.toNumber()); + console.log(rs); + console.log(Z.toNumber()); + console.log(e.toNumber()); + console.log(a.toNumber()); + console.log(r.toNumber()); + console.log(zp.toNumber()); + } + */ + const zkObject = { + inputs, + outputs, + e: e.toString('hex'), + a: a.toString('hex'), + zp: zp.toString('hex'), + importId, + }; + + // return res; + if (res) { + console.log('ZK proof successfully created and validated for event: ', eventId); + } else { + console.log('ZK proof failed for event: ', eventId); + } + return zkObject; + } + + V(e, a, Z, zp) { + return this.encrypt(this.zero, zp.fromRed().toRed(this.redSquare)) + .eq(a.redMul(Z.redPow(e))); + } +} + +module.exports = ZK; diff --git a/modules/gs1-importer.js b/modules/gs1-importer.js index 2b1c40af97..7d65f343fd 100644 --- a/modules/gs1-importer.js +++ b/modules/gs1-importer.js @@ -1,52 +1,14 @@ const { parseString } = require('xml2js'); const fs = require('fs'); const md5 = require('md5'); -const deasync = require('deasync-promise'); const xsd = require('libxml-xsd'); +const ZK = require('./ZK'); + +const zk = new ZK(); const GSInstance = require('./GraphStorageInstance'); -const utilities = require('./Utilities'); -const async = require('async'); const validator = require('validator'); -// Update import data - - -function updateImportNumber(collection, document, importId) { - const { db } = GSInstance; - return db.updateDocumentImports(collection, document, importId); -} - -/** - * Find values helper - * @param obj - * @param key - * @param list - * @return {*} - */ -function findValuesHelper(obj, key, list) { - if (!obj) return list; - if (obj instanceof Array) { - for (var i in obj) { - list = list.concat(findValuesHelper(obj[i], key, [])); - } - return list; - } - if (obj[key]) list.push(obj[key]); - - if ((typeof obj === 'object') && (obj !== null)) { - var children = Object.keys(obj); - if (children.length > 0) { - for (i = 0; i < children.length; i += 1) { - list = list.concat(findValuesHelper(obj[children[i]], key, [])); - } - } - } - return list; -} - -// sanitize - function sanitize(old_obj, new_obj, patterns) { if (typeof old_obj !== 'object') { return old_obj; } @@ -63,13 +25,6 @@ function sanitize(old_obj, new_obj, patterns) { return new_obj; } -// parsing - -function Error(message) { - console.log(`Error: ${message}`); - return message; -} - // validate function providerIdValidation(provider_id, validation_object) { const data = provider_id; @@ -216,1531 +171,762 @@ function ethWalletValidation(wallet) { return false; } +function arrayze(value) { + if (value) { + return [].concat(value); + } + return []; +} -module.exports = () => ({ - parseGS1(gs1_xml_file, callback) { - const { db } = GSInstance; - var gs1_xml = fs.readFileSync(gs1_xml_file); - - xsd.parseFile('./importers/EPCglobal-epcis-masterdata-1_2.xsd', (err, schema) => { - if (err) { - throw Error('Invalid XML structure!'); - } - - schema.validate(gs1_xml.toString(), (err, validationErrors) => { - if (err) { - throw Error('Invalid XML structure!'); - } - }); - }); - - parseString( - gs1_xml, - { explicitArray: false, mergeAttrs: true }, - /* eslint-disable consistent-return */ - async (err, result) => { - /** - * Variables - */ - - var sanitized_EPCIS_document; - - let Vocabulary_elements; - let vocabulary_element; - let inside; - var Bussines_location_elements; - let VocabularyElementList_element; - let business_location_id; - let attribute_id; - - - let data_object = {}; - let participants_data = {}; - const object_data = {}; - const batch_data = {}; - - - var sender = {}; - var receiver = {}; - var document_meta = {}; - var locations = {}; - var participants = {}; - var objects = {}; - var batches = {}; - - - const object_events = {}; - const aggregation_events = {}; - const transformation_events = {}; - - - var owned_by_edges = []; - var instance_of_edges = []; - var at_edges = []; - var read_point_edges = []; - var event_batch_edges = []; - var parent_batches_edges = []; - var child_batches_edges = []; - var input_batches_edges = []; - var output_batches_edges = []; - var business_location_edges = []; - - // READING EPCIS Document - const doc = findValuesHelper(result, 'epcis:EPCISDocument', []); - if (doc.length <= 0) { - throw Error('Missing EPCISDocument element!'); - } - - - const EPCISDocument_element = result['epcis:EPCISDocument']; - - - const creation_date_head_check = findValuesHelper(EPCISDocument_element, 'creationDate', []); - if (creation_date_head_check.length > 0) { - var temp_creation_date_head = EPCISDocument_element.creationDate; - } - const creation_date_head = temp_creation_date_head; - - var creation_date_head_validation = dateTimeValidation(creation_date_head); - if (!creation_date_head_validation) { - throw Error('Invalid Date and Time format. Please use format defined by ISO 8601 standard!'); - } +function copyProperties(from, to) { + for (const property in from) { + to[property] = from[property]; + } +} - const new_obj = {}; - sanitized_EPCIS_document = sanitize(EPCISDocument_element, new_obj, ['sbdh:', 'xmlns:']); +function parseAttributes(attributes, ignorePattern) { + const output = {}; + const inputAttributeArray = arrayze(attributes); + for (const inputElement of inputAttributeArray) { + output[inputElement.id.replace(ignorePattern, '')] = inputElement._; + } - const head = findValuesHelper(sanitized_EPCIS_document, 'EPCISHeader', []); - if (head.length <= 0) { - throw Error('Missing EPCISHeader element for EPCISDocument element!'); - } - const EPCISHeader_element = sanitized_EPCIS_document.EPCISHeader; + return output; +} +function ignorePattern(attribute, ignorePattern) { + return attribute.replace(ignorePattern, ''); +} - const standard_doc_header = findValuesHelper(EPCISHeader_element, 'StandardBusinessDocumentHeader', []); - if (standard_doc_header.length <= 0) { - throw Error('Missing StandardBusinessDocumentHeader element for EPCISHeader element!'); - } - const StandardBusinessDocumentHeader_element = - EPCISHeader_element.StandardBusinessDocumentHeader; +function parseLocations(vocabularyElementList) { + /* + { type: 'urn:ot:mda:location', + VocabularyElementList: { VocabularyElement: [Object] } } ] + */ + const locations = []; + // May be an array in VocabularyElement. + const vocabularyElementElements = arrayze(vocabularyElementList.VocabularyElement); - const document_id_check = findValuesHelper(StandardBusinessDocumentHeader_element, 'DocumentIdentification', []); - if (document_id_check.length > 0) { - var tempDocument_identification_element = - StandardBusinessDocumentHeader_element.DocumentIdentification; - } - const Document_identification_element = tempDocument_identification_element; + for (const element of vocabularyElementElements) { + const childLocations = arrayze(element.children ? element.children.id : []); - const creation_date_check = findValuesHelper(Document_identification_element, 'CreationDateAndTime', []); - if (creation_date_check.length > 0) { - var tempCreationDate_element = - Document_identification_element.CreationDateAndTime; - } - const CreationDate_element = tempCreationDate_element; + const location = { + type: 'location', + id: element.id, + attributes: parseAttributes(element.attribute, 'urn:ot:mda:location:'), + child_locations: childLocations, + }; + locations.push(location); + } - const date_validation_result = dateTimeValidation(CreationDate_element); + return locations; +} - if (!date_validation_result) { - throw Error('Invalid Date and Time format. Please use format defined by ISO 8601 standard!'); - } +function parseActors(vocabularyElementList) { + /* + { type: 'urn:ot:mda:actor', + VocabularyElementList: { VocabularyElement: [Object] } } ] + */ + const actors = []; - // //SENDER - const send = findValuesHelper(StandardBusinessDocumentHeader_element, 'Sender', []); - if (send.length <= 0) { - throw Error('Missing Sender element for StandardBusinessDocumentHeader element!'); - } - const Sender_element = StandardBusinessDocumentHeader_element.Sender; + // May be an array in VocabularyElement. + const vocabularyElementElements = arrayze(vocabularyElementList.VocabularyElement); + for (const element of vocabularyElementElements) { + const actor = { + type: 'actor', + id: element.id, + attributes: parseAttributes(element.attribute, 'urn:ot:mda:actor:'), + }; - const send_id = findValuesHelper(Sender_element, 'Identifier', []); - if (send_id.length <= 0) { - throw Error('Missing Identifier element for Sender element!'); - } - const sender_id_element = Sender_element.Identifier; + actors.push(actor); + } - const contact_info_check = findValuesHelper(Sender_element, 'ContactInformation', []); - if (contact_info_check.length > 0) { - var temp_contact_info = Sender_element.ContactInformation; - } - const contact_info_sender_element = temp_contact_info; + return actors; +} - const email_check = findValuesHelper(contact_info_sender_element, 'EmailAddress', []); - if (email_check.length > 0) { - var temp_email_check = contact_info_sender_element.EmailAddress; - } - const sender_email = temp_email_check; +function parseProducts(vocabularyElementList) { + /* + { type: 'urn:ot:mda:product', + VocabularyElementList: { VocabularyElement: [Object] } } ] + */ - const email_validation = emailValidation(sender_email); - if (!email_validation) { - throw Error('This email adress is not valid!'); - } + const products = []; + // May be an array in VocabularyElement. + const vocabularyElementElements = arrayze(vocabularyElementList.VocabularyElement); - const sendid = findValuesHelper(sender_id_element, '_', []); - if (sendid.length <= 0) { - throw Error('Missing _ element for sender_id element!'); - } - const sender_id = sender_id_element._; + for (const element of vocabularyElementElements) { + const product = { + type: 'product', + id: element.id, + attributes: parseAttributes(element.attribute, 'urn:ot:mda:product:'), + }; + products.push(product); + } - const contact_info = findValuesHelper(Sender_element, 'ContactInformation', []); - if (contact_info.length <= 0) { - throw Error('Missing ContactInformation element for Sender element!'); - } - const ContactInformation_element = Sender_element.ContactInformation; + return products; +} +function parseBatches(vocabularyElementList) { + /* + { type: 'urn:ot:mda:batch', + VocabularyElementList: { VocabularyElement: [Array] } } ] + */ - // ///RECEIVER - const receive = findValuesHelper(StandardBusinessDocumentHeader_element, 'Receiver', []); - if (receive.length <= 0) { - throw Error('Missing Receiver element for StandardBusinessDocumentHeader element!'); - } - const Receiver_element = StandardBusinessDocumentHeader_element.Receiver; + const batches = []; + // May be an array in VocabularyElement. + const vocabularyElementElements = arrayze(vocabularyElementList.VocabularyElement); - const receive_id = findValuesHelper(Receiver_element, 'Identifier', []); - if (receive_id.length <= 0) { - throw Error('Missing Identifier element for Receiver element!'); - } - const receiver_id_element = Receiver_element.Identifier; + for (const element of vocabularyElementElements) { + const batch = { + type: 'batch', + id: element.id, + attributes: parseAttributes(element.attribute, 'urn:ot:mda:batch:'), + }; - const receiver_contact_info_check = findValuesHelper(Receiver_element, 'ContactInformation', []); - if (receiver_contact_info_check.length > 0) { - var temp_contact_info_receiver = Receiver_element.ContactInformation; - } - const contact_info_receiver_element = temp_contact_info_receiver; - - const email_check_receiver = findValuesHelper(contact_info_receiver_element, 'EmailAddress', []); - if (email_check_receiver.length > 0) { - var temp_email_check_receiver = contact_info_receiver_element.EmailAddress; - } - const receiver_email = temp_email_check_receiver; + batches.push(batch); + } - const email_validation_receiver = emailValidation(receiver_email); - if (!email_validation_receiver) { - throw Error('This email adress is not valid!'); - } + return batches; +} +/** + * Create event ID + * @param senderId Sender ID + * @param event Event data + * @return {string} + */ +function getEventId(senderId, event) { + if (arrayze(event.eventTime).length === 0) { + throw Error('Missing eventTime element for event!'); + } + const event_time = event.eventTime; - const receiveid = findValuesHelper(receiver_id_element, '_', []); - if (receiveid.length <= 0) { - throw Error('Missing Identifier element for Receiver element!'); - } - const receiver_id = receiver_id_element._; + const event_time_validation = dateTimeValidation(event_time); + if (!event_time_validation) { + throw Error('Invalid date and time format for event time!'); + } + if (typeof event_time !== 'string') { + throw Error('Multiple eventTime elements found!'); + } + if (arrayze(event.eventTimeZoneOffset).length === 0) { + throw Error('Missing event_time_zone_offset element for event!'); + } + const event_time_zone_offset = event.eventTimeZoneOffset; + if (typeof event_time_zone_offset !== 'string') { + throw Error('Multiple event_time_zone_offset elements found!'); + } - const contact_info_rec = findValuesHelper(Receiver_element, 'ContactInformation', []); - if (contact_info_rec.length <= 0) { - throw Error('Missing ContactInformation element for Receiver element!'); - } - const ContactInformation_element_receiver = Receiver_element.ContactInformation; + let eventId = `${senderId}:${event_time}Z${event_time_zone_offset}`; + if (arrayze(event.baseExtension).length > 0) { + const baseExtension_element = event.baseExtension; + if (arrayze(baseExtension_element.eventID).length === 0) { + throw Error('Missing eventID in baseExtension!'); + } + eventId = baseExtension_element.eventID; + } + return eventId; +} - const doc_identification = findValuesHelper(StandardBusinessDocumentHeader_element, 'DocumentIdentification', []); - if (doc_identification.length <= 0) { - throw Error('Missing DocumentIdentification element for StandardBusinessDocumentHeader element!'); - } - const DocumentIdentification_element = - StandardBusinessDocumentHeader_element.DocumentIdentification; +function validateSender(sender) { + if (sender.EmailAddress) { + emailValidation(sender.EmailAddress); + } +} +async function processXML(err, result) { + const { db } = GSInstance; + const GLOBAL_R = 131317; + const importId = Date.now(); + + const epcisDocumentElement = result['epcis:EPCISDocument']; + + // Header stuff. + const standardBusinessDocumentHeaderElement = epcisDocumentElement.EPCISHeader['sbdh:StandardBusinessDocumentHeader']; + const senderElement = standardBusinessDocumentHeaderElement['sbdh:Sender']; + const vocabularyListElement = + epcisDocumentElement.EPCISHeader.extension.EPCISMasterData.VocabularyList; + const eventListElement = epcisDocumentElement.EPCISBody.EventList; + + // Outputs. + let locations = []; + let actors = []; + let products = []; + let batches = []; + const events = []; + const eventEdges = []; + const locationEdges = []; + const locationVertices = []; + const actorsVertices = []; + const productVertices = []; + const batchEdges = []; + const batchesVertices = []; + const eventVertices = []; + + const EDGE_KEY_TEMPLATE = 'ot_vertices/OT_KEY_'; + + const senderId = senderElement['sbdh:Identifier']._; + const sender = { + identifiers: { + id: senderId, + uid: senderElement['sbdh:Identifier']._, + }, + data: sanitize(senderElement['sbdh:ContactInformation'], {}, ['sbdh:']), + vertex_type: 'SENDER', + }; + + validateSender(sender.data); + + // Check for vocabularies. + const vocabularyElements = arrayze(vocabularyListElement.Vocabulary); + + for (const vocabularyElement of vocabularyElements) { + switch (vocabularyElement.type) { + case 'urn:ot:mda:actor': + actors = actors.concat(parseActors(vocabularyElement.VocabularyElementList)); + break; + case 'urn:ot:mda:product': + products = + products.concat(parseProducts(vocabularyElement.VocabularyElementList)); + break; + case 'urn:ot:mda:batch': + batches = + batches.concat(parseBatches(vocabularyElement.VocabularyElementList)); + break; + case 'urn:ot:mda:location': + locations = + locations.concat(parseLocations(vocabularyElement.VocabularyElementList)); + break; + default: + throw Error(`Unimplemented or unknown type: ${vocabularyElement.type}.`); + } + } - const bus_scope = findValuesHelper(StandardBusinessDocumentHeader_element, 'BusinessScope', []); - if (bus_scope.length <= 0) { - throw Error('Missing BusinessScope element for StandardBusinessDocumentHeader element!'); - } - const BusinessScope_element = - StandardBusinessDocumentHeader_element.BusinessScope; + // Check for events. + // Types: Transport, Transformation, Observation and Ownership. + for (const objectEvent of arrayze(eventListElement.ObjectEvent)) { + events.push(objectEvent); + } - sender.sender_id = {}; - sender.sender_id.identifiers = {}; - sender.sender_id.identifiers.sender_id = sender_id; - sender.sender_id.identifiers.uid = sender_id; - sender.sender_id.data = ContactInformation_element; - sender.sender_id.vertex_type = 'SENDER'; + if (eventListElement.AggregationEvent) { + for (const aggregationEvent of arrayze(eventListElement.AggregationEvent)) { + events.push(aggregationEvent); + } + } + if (eventListElement.extension && eventListElement.extension.TransformationEvent) { + for (const transformationEvent of + arrayze(eventListElement.extension.TransformationEvent)) { + events.push(transformationEvent); + } + } - receiver.receiver_id = {}; - receiver.receiver_id.identifiers = {}; - receiver.receiver_id.identifiers.receiver_id = receiver_id; - receiver.receiver_id.data = ContactInformation_element_receiver; - receiver.receiver_id.vertex_type = 'RECEIVER'; + // pre-fetch from DB. + const objectClassLocationId = await db.getClassId('Location'); + const objectClassActorId = await db.getClassId('Actor'); + const objectClassProductId = await db.getClassId('Product'); + const objectEventTransportId = await db.getClassId('Transport'); + const objectEventTransformationId = await db.getClassId('Transformation'); + const objectEventObservationId = await db.getClassId('Observation'); + const objectEventOwnershipId = await db.getClassId('Ownership'); + + for (const location of locations) { + const identifiers = { + id: location.id, + uid: location.id, + }; + const data = { + object_class_id: objectClassLocationId, + }; + + copyProperties(location.attributes, data); + + const locationKey = md5(`business_location_${senderId}_${JSON.stringify(identifiers)}_${md5(JSON.stringify(data))}`); + locationVertices.push({ + _key: locationKey, + identifiers, + data, + vertex_type: 'OBJECT', + }); - // /BUSINESS SCOPE AND DOCUMENT IDENTIFICATION + const { child_locations } = location; + for (const childId of child_locations) { + const identifiers = { + id: childId, + uid: childId, + }; + const data = { + parent_id: location.id, + }; + + const childLocationKey = md5(`child_business_location_${senderId}_${md5(JSON.stringify(identifiers))}_${md5(JSON.stringify(data))}`); + locationVertices.push({ + _key: childLocationKey, + identifiers, + data, + vertex_type: 'CHILD_BUSINESS_LOCATION', + }); - document_meta = Object.assign( - {}, - document_meta, - { BusinessScope_element, DocumentIdentification_element }, - ); + locationEdges.push({ + _key: md5(`child business_location_${senderId}_${location.id}_${JSON.stringify(identifiers)}_${md5(JSON.stringify(data))}`), + _from: `ot_vertices/${childLocationKey}`, + _to: `ot_vertices/${locationKey}`, + edge_type: 'CHILD_BUSINESS_LOCATION', + }); + } + } + for (const actor of actors) { + const identifiers = { + id: actor.id, + uid: actor.id, + }; - // ///////////READING Master Data/////////// + const data = { + object_class_id: objectClassActorId, + }; - const ext = findValuesHelper(EPCISHeader_element, 'extension', []); - if (ext.length <= 0) { - throw Error('Missing extension element for EPCISHeader element!'); - } - const extension_element = EPCISHeader_element.extension; + copyProperties(actor.attributes, data); + actorsVertices.push({ + _key: md5(`actor_${senderId}_${JSON.stringify(identifiers)}_${md5(JSON.stringify(data))}`), + _id: actor.id, + identifiers, + data, + vertex_type: 'ACTOR', + }); + } - const epcis_master = findValuesHelper(extension_element, 'EPCISMasterData', []); - if (epcis_master.length <= 0) { - throw Error('Missing EPCISMasterData element for extension element!'); - } - const EPCISMasterData_element = extension_element.EPCISMasterData; + for (const product of products) { + const identifiers = { + id: product.id, + uid: product.id, + }; + const data = { + object_class_id: objectClassProductId, + }; - const vocabulary_li = findValuesHelper(EPCISMasterData_element, 'VocabularyList', []); - if (vocabulary_li.length <= 0) { - throw Error('Missing VocabularyList element for EPCISMasterData element!'); - } - const VocabularyList_element = EPCISMasterData_element.VocabularyList; + copyProperties(product.attributes, data); + productVertices.push({ + _key: md5(`product_${senderId}_${JSON.stringify(identifiers)}_${md5(JSON.stringify(data))}`), + _id: product.id, + data, + vertex_type: 'PRODUCT', + }); + } - const vocabulary = findValuesHelper(VocabularyList_element, 'Vocabulary', []); - if (vocabulary.length <= 0) { - throw Error('Missing Vocabulary element for VocabularyList element!'); - } - Vocabulary_elements = VocabularyList_element.Vocabulary; + for (const batch of batches) { + const productId = batch.attributes.productid; + const identifiers = { + id: batch.id, + uid: batch.id, + }; - if (!(Vocabulary_elements instanceof Array)) { - const temp_vocabulary_elements = Vocabulary_elements; - Vocabulary_elements = []; - Vocabulary_elements.push(temp_vocabulary_elements); - } + const data = { + parent_id: productId, + }; + copyProperties(batch.attributes, data); - for (const i in Vocabulary_elements) { - vocabulary_element = Vocabulary_elements[i]; + const key = md5(`batch_${senderId}_${JSON.stringify(identifiers)}_${md5(JSON.stringify(data))}`); + batchesVertices.push({ + _key: key, + identifiers: { + id: batch.id, + uid: batch.id, + }, + data, + vertex_type: 'BATCH', + }); - if (!(vocabulary_element instanceof Array)) { - const temp_vocabularyel_elements = vocabulary_element; - vocabulary_element = []; - vocabulary_element.push(temp_vocabularyel_elements); - } + batchEdges.push({ + _key: md5(`child business_location_${senderId}_${batch.id}_${productId}`), + _from: `ot_vertices/${key}`, + _to: `${EDGE_KEY_TEMPLATE + productId}`, + edge_type: 'IS', + }); + } - for (let j in vocabulary_element) { - inside = vocabulary_element[j]; - let pro; + // Store vertices in db. Update versions - for (j in inside) { - pro = inside[j]; - const typ = findValuesHelper(pro, 'type', []); - if (typ.length <= 0) { - throw Error('Missing type element for element!'); - } - const v_type = pro.type; + function getClassId(event) { + // TODO: Support all other types. + if (event.action && event.action === 'OBSERVE') { + return objectEventObservationId; + } + return objectEventTransformationId; + } - // ////////BUSINESS_LOCATION///////////// - if (v_type === 'urn:epcglobal:epcis:vtype:BusinessLocation') { - Bussines_location_elements = pro; + // TODO handle extensions + for (const event of events) { + const eventId = getEventId(senderId, event); - const voc_el_list = findValuesHelper(Bussines_location_elements, 'VocabularyElementList', []); - if (voc_el_list.length === 0) { - throw Error('Missing VocabularyElementList element for element!'); - } - VocabularyElementList_element = - Bussines_location_elements.VocabularyElementList; - - - for (const k in VocabularyElementList_element) { - data_object = {}; - - const VocabularyElement_element = - VocabularyElementList_element[k]; - // console.log(VocabularyElement_element) - - for (const x in VocabularyElement_element) { - const v = VocabularyElement_element[x]; - // console.log(v) - - const loc_id = findValuesHelper(v, 'id', []); - if (loc_id.length <= 0) { - throw Error('Missing id element for VocabularyElement element!'); - } - const str = v.id; - business_location_id = str.replace('urn:epc:id:sgln:', ''); - - - const attr = findValuesHelper(v, 'attribute', []); - if (attr.length <= 0) { - throw Error('Missing attribute element for VocabularyElement element!'); - } - const { attribute } = v; - - - for (const y in attribute) { - const kk = attribute[y]; + const { extension } = event; + let eventCategories; + if (extension.extension) { + const eventClass = extension.extension.OTEventClass; + eventCategories = arrayze(eventClass).map(obj => ignorePattern(obj, 'ot:events:')); + } else { + const eventClass = extension.OTEventClass; + eventCategories = arrayze(eventClass).map(obj => ignorePattern(obj, 'ot:event:')); + } - const att_id = findValuesHelper(kk, 'id', []); - if (att_id.length <= 0) { - throw Error('Missing id attribute for element!'); - } - attribute_id = kk.id; - - const attribute_value = kk._; - - - if (attribute_id === 'urn:ts:location:country') { - const country_validation = - countryValidation(attribute_value); - if (!country_validation) { - throw Error('Invalid country code. Please use two characters!'); - } - } - - if (attribute_id === 'urn:ts:location:postalCode') { - const postal_code_validation = - postalCodeValidation(attribute_value); - if (!postal_code_validation) { - throw Error('Invalid postal code!'); - } - } - - - data_object[attribute_id] = kk._; - } - - const children_check = findValuesHelper(v, 'children', []); - if (children_check.length === 0) { - throw Error('Missing children element for element!'); - } - const children_elements = v.children; - - - if (findValuesHelper(children_elements, 'id', []).length === 0) { - throw Error('Missing id element in children element for business location!'); - } - - const children_id = children_elements.id; - var child_id_obj; - var child_location_id; - for (const mn in children_id) { - child_id_obj = (children_id[mn]); - - if (!(child_id_obj instanceof Array)) { - const temp_child_id = child_id_obj; - child_id_obj = []; - child_id_obj.push(temp_child_id); - } - - for (const r in child_id_obj) { - child_location_id = child_id_obj[r]; - - business_location_edges.push({ - _key: md5(`child_business_location_${sender_id}_${child_location_id}_${business_location_id}`), - _from: `ot_vertices/${md5(`business_location_${sender_id}_${child_location_id}`)}`, - _to: `ot_vertices/${md5(`business_location_${sender_id}_${business_location_id}`)}`, - edge_type: 'CHILD_BUSINESS_LOCATION', - }); - - locations[child_location_id] = {}; - locations[child_location_id].data = { type: 'child_location' }; - locations[child_location_id].identifiers = {}; - locations[child_location_id] - .identifiers - .bussines_location_id = child_location_id; - locations[child_location_id] - .identifiers.uid = child_location_id; - locations[child_location_id].vertex_type = 'BUSINESS_LOCATION'; - locations[child_location_id]._key = md5(`business_location_${sender_id}_${child_location_id}`); - } - } - - if (findValuesHelper(v, 'extension', []).length !== 0) { - const attr = findValuesHelper(v.extension, 'attribute', []); - if (attr.length !== 0) { - let ext_attribute; - ext_attribute = v.extension.attribute; - - if (ext_attribute.length === undefined) { - ext_attribute = [ext_attribute]; - } - - for (const y in ext_attribute) { - const kk = ext_attribute[y]; - - - const att_id = findValuesHelper(kk, 'id', []); - if (att_id.length <= 0) { - throw Error('Missing id attribute for element!'); - } - attribute_id = kk.id; - - - attribute_id = attribute_id.replace('urn:ot:location:', ''); - - - if (attribute_id === 'participantId') { - owned_by_edges.push({ - _from: `ot_vertices/${md5(`business_location_${sender_id}_${business_location_id}`)}`, - _to: `ot_vertices/${md5(`participant_${sender_id}_${kk._}`)}`, - edge_type: 'OWNED_BY', - _key: md5(`owned_by_${sender_id}_${business_location_id}_${kk._}`), - }); - } - } - } - } - - const new_obj = {}; - const sanitized_object_data = sanitize(object_data, new_obj, ['urn:', 'ot:', 'mda:', 'object:']); - - locations[business_location_id] = {}; - locations[business_location_id].identifiers = {}; - locations[business_location_id] - .identifiers - .bussines_location_id = business_location_id; - locations[business_location_id] - .identifiers.uid = business_location_id; - locations[business_location_id] - .data = utilities.copyObject(sanitized_object_data); - locations[business_location_id].vertex_type = 'BUSINESS_LOCATION'; - locations[business_location_id]._key = md5(`business_location_${sender_id}_${business_location_id}`); - } - } - } else if (v_type === 'urn:ot:mda:participant') { - let participant_id; - // /////PARTICIPANT/////////// - const Participant_elements = pro; - - const extension_check = findValuesHelper(Participant_elements, 'extension', []); - if (extension_check.length === 0) { - throw Error('Missing extension element for Participant element!'); - } - const exten_element = Participant_elements.extension; - - - const ot_voc_check = findValuesHelper(exten_element, 'OTVocabularyElement', []); - if (ot_voc_check.length === 0) { - throw Error('Missing OTVocabularyElement for extension element!'); - } - const OTVocabularyElement_element = - exten_element.OTVocabularyElement; - - let temp_participant_id; - const participant_id_check = findValuesHelper(OTVocabularyElement_element, 'id', []); - if (participant_id_check.length === 0) { - throw Error('Missing id for Participant element!'); - } else { - temp_participant_id = OTVocabularyElement_element.id; - } - - if (!temp_participant_id.includes('urn:ot:mda:participant', 0) === true) { - throw Error('Invalid Participant ID'); - } else { - participant_id = temp_participant_id; - } - - const attribute_check = findValuesHelper(OTVocabularyElement_element, 'attribute', []); - if (attribute_check.length === 0) { - throw Error('Missing attribute for Participant element!'); - } - const attribute_elements = OTVocabularyElement_element.attribute; - - // console.log(OTVocabularyElement_element) - - participants_data = {}; - - for (const zx in attribute_elements) { - const attribute_el = attribute_elements[zx]; - - const value_check = findValuesHelper(attribute_el, '_', []); - if (value_check.length === 0) { - throw Error('Missing value for attribute element!'); - } - const value = attribute_el._; - - - let attr_id; - let temp_attr_id; - const attr_id_check = findValuesHelper(attribute_el, 'id', []); - if (attr_id_check.length === 0) { - throw Error('Missing id element for attribute element!'); - } else { - temp_attr_id = attribute_el.id; - } - if (!temp_attr_id.includes('urn:ot:mda:participant', 0) === true) { - throw Error('Invalid Attribute ID'); - } else { - attr_id = temp_attr_id.replace('urn:ot:mda:participant:', ''); - } - - participants_data[attr_id] = value; - } - - participants[participant_id] = {}; - participants[participant_id].identifiers = {}; - participants[participant_id] - .identifiers - .participant_id = participant_id; - participants[participant_id].identifiers.uid = participant_id; - participants[participant_id] - .data = utilities.copyObject(participants_data); - participants[participant_id].vertex_type = 'PARTICIPANT'; - participants[participant_id]._key = md5(`participant_${sender_id}_${participant_id}`); - } else if (v_type === 'urn:ot:mda:object') { - const Object_elements = pro; - // ////OBJECT//////// - const extensio_check = findValuesHelper(Object_elements, 'extension', []); - if (extensio_check.length === 0) { - throw Error('Missing extension element for Object element!'); - } - const extensio_element = Object_elements.extension; - - - const OTVocabularyEl_check = findValuesHelper(extensio_element, 'OTVocabularyElement', []); - if (OTVocabularyEl_check.length === 0) { - throw Error('Missing OTVocabularyElement element for extension element!'); - } - const OTVocabularyEl = extensio_element.OTVocabularyElement; - - - const object_id_check = findValuesHelper(OTVocabularyEl, 'id', []); - if (object_id_check.length === 0) { - throw Error('Missing id element for OTVocabularyElement!'); - } - var object_id = OTVocabularyEl.id; - - - const attribute_el_check = findValuesHelper(OTVocabularyEl, 'attribute', []); - if (attribute_el_check.length === 0) { - throw Error('Missing attribute element for OTVocabularyElement!'); - } - const object_attribute_elements = OTVocabularyEl.attribute; - - - for (const rr in object_attribute_elements) { - const single_attribute = object_attribute_elements[rr]; - - let temp_single_attribute_id; - let single_attribute_id; - const single_attribute_id_check = findValuesHelper(single_attribute, 'id', []); - if (single_attribute_id_check.length === 0) { - throw Error('Missing id element for attribute element!'); - } else { - temp_single_attribute_id = single_attribute.id; - } - - if (!temp_single_attribute_id.includes('urn:ot:mda:object:', 0) === true) { - throw Error('Invalid Attribute ID'); - } else { - single_attribute_id = temp_single_attribute_id; - } - - - // console.log(temp_single_attribute_id) - - const single_attribute_value_check = - findValuesHelper(single_attribute, '_', []); - if (single_attribute_value_check.length === 0) { - throw Error('Missing value element for attribute element!'); - } - const single_attribute_value = single_attribute._; - - - if (single_attribute_id === 'urn:ot:mda:object:ean13') { - const ean13_validation = - ean13Validation(single_attribute_value); - // console.log(ean13_validation) - if (!ean13_validation) { - throw Error('EAN13 code is not valid!'); - } - } - - - object_data[single_attribute_id] = single_attribute_value; - const new_obj = {}; - const sanitized_object_data = sanitize(object_data, new_obj, ['urn:', 'ot:', 'mda:', 'object:']); - - - objects[object_id] = {}; - objects[object_id].identifiers = {}; - objects[object_id].identifiers.object_id = object_id; - objects[object_id] - .data = utilities.copyObject(sanitized_object_data); - objects[object_id].vertex_type = 'OBJECT'; - objects[object_id]._key = md5(`object_${sender_id}_${object_id}`); - } - } else if (v_type === 'urn:ot:mda:batch') { - const Batch_elements = pro; - // //////BATCH///////// - - const batch_extension_check = findValuesHelper(Batch_elements, 'extension', []); - if (batch_extension_check.length === 0) { - throw Error('Missing extension element for Batch element!'); - } - const batch_extension = Batch_elements.extension; - - - const OTVoc_El_elements_check = findValuesHelper(batch_extension, 'OTVocabularyElement', []); - if (OTVoc_El_elements_check.length === 0) { - throw Error('Missing OTVocabularyElement element for extension element!'); - } - const OTVoc_El_elements = batch_extension.OTVocabularyElement; - - - let ot_vocabulary_element; - for (const g in OTVoc_El_elements) { - let object_id_instance = false; - let valid_attribute = false; - ot_vocabulary_element = OTVoc_El_elements[g]; - - const batch_id_element_check = findValuesHelper(ot_vocabulary_element, 'id', []); - if (batch_id_element_check.length === 0) { - throw Error('Missing id element for OTVocabularyElement!'); - } - const batch_id = ot_vocabulary_element.id; - - - const batch_attribute_el_check = findValuesHelper(ot_vocabulary_element, 'attribute', []); - if (batch_attribute_el_check.length === 0) { - throw Error('Missing attribute element for OTVocabularyElement!'); - } - const batch_attribute_el = ot_vocabulary_element.attribute; - - - let single; - for (const one in batch_attribute_el) { - single = batch_attribute_el[one]; - - var temp_batch_attribute_id; - const batch_attribute_id_check = findValuesHelper(single, 'id', []); - if (batch_attribute_id_check.length === 0) { - throw Error('Missing id element for attribute element!'); - } else { - temp_batch_attribute_id = single.id; - } - - - if (temp_batch_attribute_id.includes('urn:ot:mda:batch:objectid', 0) && object_id_instance === false) { - object_id_instance = true; - } else if (temp_batch_attribute_id.includes('urn:ot:mda:batch:', 0)) { - valid_attribute = true; - } else { - throw Error('Invalid Attribute ID'); - } - - const batch_attribute_value_check = findValuesHelper(single, '_', []); - if (batch_attribute_value_check.length === 0) { - throw Error('Missing value element for attribute element!'); - } - const batch_attribute_value = single._; - - if (temp_batch_attribute_id === 'urn:ot:mda:batch:productiondate') { - const production_date_validation = - dateTimeValidation(batch_attribute_value); - if (!production_date_validation) { - throw Error('Invalid date and time format for production date!'); - } - } - - if (temp_batch_attribute_id === 'urn:ot:mda:batch:expirationdate') { - const expiration_date_validation = - dateTimeValidation(batch_attribute_value); - if (!expiration_date_validation) { - throw Error('Invalid date and time format for expiration date!'); - } - } - - - batch_data[temp_batch_attribute_id] = batch_attribute_value; - - const new_obj = {}; - const sanitized_batch_data = sanitize(batch_data, new_obj, ['urn:', 'ot:', 'mda:', 'batch:']); - - if (sanitized_batch_data.objectid !== undefined) { - instance_of_edges.push({ - _from: `ot_vertices/${md5(`batch_${sender_id}_${batch_id}`)}`, - _to: `ot_vertices/${md5(`object_${sender_id}_${object_id}`)}`, - _key: md5(`object_${sender_id}__${batch_id}${object_id}`), - edge_type: 'INSTANCE_OF', - }); - } - - - batches[batch_id] = {}; - batches[batch_id].identifiers = {}; - batches[batch_id].identifiers.batch_id = batch_id; - batches[batch_id].identifiers.uid = batch_id; - batches[batch_id] - .data = utilities.copyObject(sanitized_batch_data); - batches[batch_id].vertex_type = 'BATCH'; - batches[batch_id]._key = md5(`batch_${sender_id}_${batch_id}`); - } - - - // console.log(valid_attribute) - - if (!object_id_instance) { - throw Error('Missing Object ID'); - } else if (valid_attribute) { - // batch_attribute_id = temp_batch_attribute_id; - } else { - throw Error('Invalid Attribute ID'); - } - } - } else { - throw Error('Invalid Vocabulary type'); - } - } + const identifiers = { + id: eventId, + uid: eventId, + }; + + let inputQuantities = []; + const outputQuantities = []; + if (eventCategories.includes('Ownership') || eventCategories.includes('Transport') + || eventCategories.includes('Observation')) { + const bizStep = ignorePattern(event.bizStep, 'urn:epcglobal:cbv:bizstep:'); + + const { quantityList } = extension; + if (bizStep === 'shipping') { + const tmpOutputQuantities = arrayze(quantityList.quantityElement) + .map(elem => ({ + object: elem.epcClass, + quantity: parseInt(elem.quantity, 10), + })); + for (const outputQ of tmpOutputQuantities) { + // eslint-disable-next-line + const vertex = await db.getVertexWithMaxVersion(outputQ.object); + if (vertex) { + const quantities = vertex.data.quantities.private; + const quantity = { + object: outputQ.object, + quantity: parseInt(quantities.quantity, 10), + r: quantities.r, + }; + inputQuantities.push(quantity); + outputQuantities.push(quantity); + } else { + inputQuantities.push({ + added: true, + object: outputQ.object, + quantity: parseInt(outputQ.quantity, 10), + }); + outputQuantities.push({ + added: true, + object: outputQ.object, + quantity: parseInt(outputQ.quantity, 10), + r: GLOBAL_R, + }); } } - - // READING EPCIS Document Body - - if (findValuesHelper(EPCISDocument_element, 'EPCISBody', []).length !== 0) { - const body_element = EPCISDocument_element.EPCISBody; - - if (findValuesHelper(result, 'EventList', []).length === 0) { - throw Error('Missing EventList element'); - } - - var event_list_element = body_element.EventList; - - - for (var event_type in event_list_element) { - let events = []; - - if (event_list_element[event_type].length === undefined) { - events = [event_list_element[event_type]]; - } else { - events = event_list_element[event_type]; - } - - - for (const i in events) { - let event_batches = []; - - const event = events[i]; - - if (event_type === 'ObjectEvent') { - // eventTime - if (findValuesHelper(event, 'eventTime', []).length === 0) { - throw Error('Missing eventTime element for event!'); - } - - const event_time = event.eventTime; - - var event_time_validation = dateTimeValidation(event_time); - if (!event_time_validation) { - throw Error('Invalid date and time format for event time!'); - } - - if (typeof event_time !== 'string') { - throw Error('Multiple eventTime elements found!'); - } - - // eventTimeZoneOffset - if (findValuesHelper(event, 'eventTimeZoneOffset', []).length === 0) { - throw Error('Missing event_time_zone_offset element for event!'); - } - - const event_time_zone_offset = event.eventTimeZoneOffset; - - if (typeof event_time_zone_offset !== 'string') { - throw Error('Multiple event_time_zone_offset elements found!'); - } - - let event_id = `${sender_id}:${event_time}Z${event_time_zone_offset}`; - - // baseExtension + eventID - if (findValuesHelper(event, 'baseExtension', []).length > 0) { - const baseExtension_element = event.baseExtension; - - - if (findValuesHelper(baseExtension_element, 'eventID', []).length === 0) { - throw Error('Missing eventID in baseExtension!'); - } - - event_id = baseExtension_element.eventID; - } - - // epcList - if (findValuesHelper(event, 'epcList', []).length === 0) { - throw Error('Missing epcList element for event!'); - } - - const { epcList } = event; - - if (findValuesHelper(epcList, 'epc', []).length === 0) { - throw Error('Missing epc element in epcList for event!'); - } - - const { epc } = epcList; - - if (typeof epc === 'string') { - event_batches = [epc]; - } else { - event_batches = epc; - } - - // readPoint - let read_point; - if (findValuesHelper(event, 'readPoint', []).length !== 0) { - const read_point_element = event.readPoint; - - if (findValuesHelper(read_point_element, 'id', []).length === 0) { - throw Error('Missing id for readPoint!'); - } - - read_point = read_point_element.id; - } - - - // bizLocation - let biz_location; - if (findValuesHelper(event, 'bizLocation', []).length !== 0) { - const biz_location_element = event.bizLocation; - - if (findValuesHelper(biz_location_element, 'id', []).length === 0) { - throw Error('Missing id for bizLocation!'); - } - - biz_location = biz_location_element.id; - } - - // extension - if (findValuesHelper(event, 'extension', []).length !== 0) { - const obj_event_extension_element = event.extension; - const quantityElement_element = - obj_event_extension_element.quantityList.quantityElement; - const extensionElement_extension = - obj_event_extension_element.extension; - - for (const element in quantityElement_element) { - const single_element = quantityElement_element[element]; - - const { quantity } = single_element; - - const quantity_validation = numberValidation(quantity); - if (!quantity_validation) { - throw Error('Invalid format for quantity element!'); - } - } - - - for (const element in extensionElement_extension) { - const temperature = extensionElement_extension[element]; - - const temperature_validation = - numberValidation(temperature); - if (!temperature_validation) { - throw Error('Invalid format for temperature element!'); - } - } - } - - const object_event = { - identifiers: { - event_id, - uid: event_id, - }, - data: event, - vertex_type: 'EVENT', - _key: md5(`event_${sender_id}_${event_id}`), - }; - - object_events[event_id] = utilities.copyObject(object_event); - - for (const bi in event_batches) { - event_batch_edges.push({ - _key: md5(`event_batch_${sender_id}_${event_id}_${event_batches[bi]}`), - _from: `ot_vertices/${md5(`batch_${sender_id}_${event_batches[bi]}`)}`, - _to: `ot_vertices/${md5(`event_${sender_id}_${event_id}`)}`, - edge_type: 'EVENT_BATCHES', - }); - } - - if (read_point !== undefined) { - read_point_edges.push({ - _key: md5(`read_point_${sender_id}_${event_id}_${read_point}`), - _from: `ot_vertices/${md5(`event_${sender_id}_${event_id}`)}`, - _to: `ot_vertices/${md5(`business_location_${sender_id}_${read_point}`)}`, - edge_type: 'READ_POINT', - }); - } - - if (biz_location !== undefined) { - at_edges.push({ - _key: md5(`at_${sender_id}_${event_id}_${biz_location}`), - _from: `ot_vertices/${md5(`event_${sender_id}_${event_id}`)}`, - _to: `ot_vertices/${md5(`business_location_${sender_id}_${biz_location}`)}`, - edge_type: 'AT', - }); - } - } else if (event_type === 'AggregationEvent') { - // eventTime - if (findValuesHelper(event, 'eventTime', []).length === 0) { - throw Error('Missing eventTime element for event!'); - } - - const event_time = event.eventTime; - - if (typeof event_time !== 'string') { - throw Error('Multiple eventTime elements found!'); - } - - // eventTimeZoneOffset - if (findValuesHelper(event, 'eventTimeZoneOffset', []).length === 0) { - throw Error('Missing event_time_zone_offset element for event!'); - } - - const event_time_zone_offset = event.eventTimeZoneOffset; - - if (typeof event_time_zone_offset !== 'string') { - throw Error('Multiple event_time_zone_offset elements found!'); - } - - let event_id = `${sender_id}:${event_time}Z${event_time_zone_offset}`; - - // baseExtension + eventID - if (findValuesHelper(event, 'baseExtension', []).length > 0) { - const baseExtension_element = event.baseExtension; - - - if (findValuesHelper(baseExtension_element, 'eventID', []).length === 0) { - throw Error('Missing eventID in baseExtension!'); - } - - event_id = baseExtension_element.eventID; - } - - // parentID - if (findValuesHelper(event, 'parentID', []).length === 0) { - throw Error('Missing parentID element for Aggregation event!'); - } - - const parent_id = event.parentID; - - // childEPCs - let child_epcs = []; - - if (findValuesHelper(event, 'childEPCs', []).length === 0) { - throw Error('Missing childEPCs element for event!'); - } - - const epcList = event.childEPCs; - - if (findValuesHelper(epcList, 'epc', []).length === 0) { - throw Error('Missing epc element in epcList for event!'); - } - - const { epc } = epcList; - - if (typeof epc === 'string') { - child_epcs = [epc]; - } else { - child_epcs = epc; - } - - // readPoint - let read_point; - if (findValuesHelper(event, 'readPoint', []).length !== 0) { - const read_point_element = event.readPoint; - - if (findValuesHelper(read_point_element, 'id', []).length === 0) { - throw Error('Missing id for readPoint!'); - } - - read_point = read_point_element.id; - } - - // bizLocation - let biz_location; - if (findValuesHelper(event, 'bizLocation', []).length !== 0) { - const biz_location_element = event.bizLocation; - - if (findValuesHelper(biz_location_element, 'id', []).length === 0) { - throw Error('Missing id for bizLocation!'); - } - - biz_location = biz_location_element.id; - } - - const aggregation_event = { - identifiers: { - event_id, - uid: event_id, - }, - data: event, - vertex_type: 'EVENT', - _key: md5(`event_${sender_id}_${event_id}`), - }; - - - aggregation_events[event_id] = - utilities.copyObject(aggregation_event); - - for (const bi in child_epcs) { - child_batches_edges.push({ - _key: md5(`child_batch_${sender_id}_${event_id}_${child_epcs[bi]}`), - _from: `ot_vertices/${md5(`event_${sender_id}_${event_id}`)}`, - _to: `ot_vertices/${md5(`batch_${sender_id}_${child_epcs[bi]}`)}`, - edge_type: 'CHILD_BATCH', - }); - } - - if (read_point !== undefined) { - read_point_edges.push({ - _key: md5(`read_point_${sender_id}_${event_id}_${read_point}`), - _from: `ot_vertices/${md5(`event_${sender}_${event_id}`)}`, - _to: `ot_vertices/${md5(`business_location_${sender_id}_${read_point}`)}`, - edge_type: 'READ_POINT', - - }); - } - - if (biz_location !== undefined) { - at_edges.push({ - _key: md5(`at_${sender_id}_${event_id}_${biz_location}`), - _from: `ot_vertices/${md5(`event_${sender_id}_${event_id}`)}`, - _to: `ot_vertices/${md5(`business_location_${sender_id}_${biz_location}`)}`, - edge_type: 'AT', - }); - } - - parent_batches_edges.push({ - _key: md5(`at_${sender_id}_${event_id}_${biz_location}`), - _from: `ot_vertices/${md5(`batch_${sender_id}_${parent_id}`)}`, - _to: `ot_vertices/${md5(`event_${sender_id}_${event_id}`)}`, - edge_type: 'PARENT_BATCH', - }); - } else if (event_type === 'extension') { - const extension_events = event; - - for (var ext_event_type in extension_events) { - let ext_events = []; - - if (extension_events[ext_event_type].length === undefined) { - ext_events = [extension_events[ext_event_type]]; - } else { - ext_events = event_list_element[ext_event_type]; - } - - - for (const i in ext_events) { - const ext_event = ext_events[i]; - - if (ext_event_type === 'TransformationEvent') { - // eventTime - if (findValuesHelper(ext_event, 'transformationID', []).length === 0) { - throw Error('Missing transformationID element for event!'); - } - - const ext_event_id = ext_event.transformationID; - - // inputEPCList - let input_epcs = []; - - if (findValuesHelper(ext_event, 'inputEPCList', []).length === 0) { - throw Error('Missing inputEPCList element for event!'); - } - - const epcList = ext_event.inputEPCList; - - if (findValuesHelper(epcList, 'epc', []).length === 0) { - throw Error('Missing epc element in epcList for event!'); - } - - const { epc } = epcList; - - if (typeof epc === 'string') { - input_epcs = [epc]; - } else { - input_epcs = epc; - } - - // outputEPCList - let output_epcs = []; - - if (findValuesHelper(ext_event, 'outputEPCList', []).length !== 0) { - const epcList = ext_event.outputEPCList; - - if (findValuesHelper(epcList, 'epc', []).length === 0) { - throw Error('Missing epc element in epcList for event!'); - } - - const { epc } = epcList; - - if (typeof epc === 'string') { - output_epcs = [epc]; - } else { - output_epcs = epc; - } - } - - - // readPoint - let read_point; - if (findValuesHelper(ext_event, 'readPoint', []).length !== 0) { - const read_point_element = ext_event.readPoint; - - if (findValuesHelper(read_point_element, 'id', []).length === 0) { - throw Error('Missing id for readPoint!'); - } - - read_point = read_point_element.id; - } - - const transformation_event = { - identifiers: { - event_id: ext_event_id, - uid: ext_event_id, - }, - data: ext_event, - vertex_type: 'EVENT', - _key: md5(`event_${sender_id}_${ext_event_id}`), - }; - - transformation_events[ext_event_id] = - utilities.copyObject(transformation_event); - - // bizLocation - let biz_location; - if (findValuesHelper( - ext_event, - 'bizLocation', - [], - ).length !== 0) { - const biz_location_element = - ext_event.bizLocation; - - if (findValuesHelper(biz_location_element, 'id', []).length === 0) { - throw Error('Missing id for bizLocation!'); - } - - biz_location = biz_location_element.id; - } - - - for (const bi in input_epcs) { - input_batches_edges.push({ - _key: md5(`child_batch_${sender_id}_${ext_event_id}_${input_epcs[bi]}`), - _from: `ot_vertices/${md5(`event_${sender_id}_${ext_event_id}`)}`, - _to: `ot_vertices/${md5(`batch_${sender_id}_${input_epcs[bi]}`)}`, - edge_type: 'INPUT_BATCH', - }); - } - - for (const bi in output_epcs) { - output_batches_edges.push({ - _key: md5(`child_batch_${sender_id}_${ext_event_id}_${output_epcs[bi]}`), - _from: `ot_vertices/${md5(`batch_${sender_id}_${output_epcs[bi]}`)}`, - _to: `ot_vertices/${md5(`event_${sender_id}_${ext_event_id}`)}`, - edge_type: 'OUTPUT_BATCH', - }); - } - - if (read_point !== undefined) { - read_point_edges.push({ - _key: md5(`read_point_${sender_id}_${ext_event_id}_${read_point}`), - _from: `ot_vertices/${md5(`event_${sender_id}_${ext_event_id}`)}`, - _to: `ot_vertices/${md5(`business_location_${sender_id}_${read_point}`)}`, - edge_type: 'READ_POINT', - - }); - } - - if (biz_location !== undefined) { - at_edges.push({ - _key: md5(`at_${sender_id}_${ext_event_id}_${biz_location}`), - _from: `ot_vertices/${md5(`event_${sender_id}_${ext_event_id}`)}`, - _to: `ot_vertices/${md5(`business_location_${sender_id}_${biz_location}`)}`, - edge_type: 'AT', - }); - } - } else { - throw Error(`Unsupported event type: ${event_type}`); - } - } - } - } else { - throw Error(`Unsupported event type: ${event_type}`); - } - } - } - - const vertices_list = []; - const edges_list = []; - const import_id = Date.now(); - - const temp_participants = []; - for (const i in participants) { - temp_participants.push(participants[i]); - vertices_list.push(participants[i]); - } - - try { - deasync(db.createCollection('ot_vertices')); - deasync(db.createEdgeCollection('ot_edges')); - } catch (err) { - console.log(err); - } - - async.each(temp_participants, (participant, next) => { - db.addDocument('ot_vertices', participant).then(() => { - updateImportNumber('ot_vertices', participant, import_id).then(() => { - next(); - }); + } else { + inputQuantities = arrayze(quantityList.quantityElement).map(elem => ({ + object: elem.epcClass, + quantity: parseInt(elem.quantity, 10), + r: GLOBAL_R, + })); + for (const inputQ of inputQuantities) { + // eslint-disable-next-line + const vertex = await db.getVertexWithMaxVersion(inputQ.object); + if (vertex) { + const quantities = vertex.data.quantities.private; + outputQuantities.push({ + object: inputQ.object, + quantity: parseInt(quantities.quantity, 10), + r: quantities.r, }); - }, () => { - console.log('Writing participants complete'); - }); - - const temp_objects = []; - for (const i in objects) { - temp_objects.push(objects[i]); - vertices_list.push(objects[i]); - } - - async.each(temp_objects, (object, next) => { - db.addDocument('ot_vertices', object).then(() => { - updateImportNumber('ot_vertices', object, import_id).then(() => { - next(); - }); + } else { + outputQuantities.push({ + added: true, + object: inputQ.object, + quantity: parseInt(inputQ.quantity, 10), }); - }, () => { - console.log('Writing objects complete'); - }); - - const temp_locations = []; - for (const i in locations) { - temp_locations.push(locations[i]); - vertices_list.push(locations[i]); } - - async.each(temp_locations, (location, next) => { - db.addDocument('ot_vertices', location).then(() => { - updateImportNumber('ot_vertices', location, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing business locations complete'); + } + } + } else { + const { inputQuantityList, outputQuantityList } = event; + for (const inputQuantity of inputQuantityList) { + // eslint-disable-next-line + const vertex = await db.getVertexWithMaxVersion(inputQuantity.object); + if (vertex) { + const quantities = vertex.data.quantities.private; + const quantity = { + object: inputQuantity.object, + quantity: parseInt(quantities.quantity, 10), + r: quantities.r, + }; + inputQuantities.push(quantity); + } else { + inputQuantities.push({ + added: true, + object: inputQuantity.object, + quantity: parseInt(inputQuantity.quantity, 10), }); - - const temp_batches = []; - for (const i in batches) { - temp_batches.push(batches[i]); - vertices_list.push(batches[i]); + } + } + for (const outputQuantity of outputQuantityList) { + // eslint-disable-next-line + const vertex = await db.getVertexWithMaxVersion(outputQuantity.object); + if (vertex) { + const quantities = vertex.data.quantities.private; + const quantity = { + object: outputQuantity.object, + quantity: parseInt(quantities.quantity, 10), + r: quantities.r, + }; + outputQuantities.push(quantity); + } else { + outputQuantities.push({ + added: true, + object: outputQuantity.object, + quantity: parseInt(outputQuantity.quantity, 10), + }); + } + } + } + const quantities = zk.P(importId, eventId, inputQuantities, outputQuantities); + for (const quantity of quantities.inputs.concat(quantities.outputs)) { + if (quantity.added) { + delete quantity.added; + let batchFound = false; + for (const batch of batchesVertices) { + if (batch.identifiers.uid === quantity.object) { + batchFound = true; + batch.data.quantities = quantity; + batch._key = md5(`batch_${senderId}_${batch.data}`); + break; } + } + if (!batchFound) { + throw new Error(`Invalid import! Batch ${quantity.object} not found.`); + } + } + } + event.quantities = quantities; - async.each(temp_batches, (batch, next) => { - db.addDocument('ot_vertices', batch).then(() => { - updateImportNumber('ot_vertices', batch, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing batches complete'); - }); + const data = { + object_class_id: getClassId(event), + vertex_type: 'EVENT', + categories: eventCategories, + }; + copyProperties(event, data); - const temp_object_events = []; - for (const i in object_events) { - temp_object_events.push(object_events[i]); - vertices_list.push(object_events[i]); - } + const eventKey = md5(`event_${senderId}_${JSON.stringify(identifiers)}_${md5(JSON.stringify(data))}`); + eventVertices.push({ + _key: eventKey, + data, + identifiers, + }); - async.each(temp_object_events, (event, next) => { - db.addDocument('ot_vertices', event).then(() => { - updateImportNumber('ot_vertices', event, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing object events complete'); + if (extension.extension) { + if (extension.extension.sourceList) { + const sources = arrayze(extension.extension.sourceList.source._); + for (const source of sources) { + eventEdges.push({ + _key: md5(`source_${senderId}_${eventId}_${source}`), + _from: `ot_vertices/${eventKey}`, + _to: `${EDGE_KEY_TEMPLATE + source}`, + edge_type: 'SOURCE', }); + } + } - const temp_aggregation_events = []; - for (const i in aggregation_events) { - temp_aggregation_events.push(aggregation_events[i]); - vertices_list.push(aggregation_events[i]); - } - - async.each(temp_aggregation_events, (event, next) => { - db.addDocument('ot_vertices', event).then(() => { - updateImportNumber('ot_vertices', event, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing aggregation events complete'); + if (extension.extension.destinationList) { + const destinations = arrayze(extension.extension.destinationList.destination._); + for (const destination of destinations) { + eventEdges.push({ + _key: md5(`destination_${senderId}_${eventId}_${destination}`), + _from: `ot_vertices/${eventKey}`, + _to: `${EDGE_KEY_TEMPLATE + destination}`, + edge_type: 'DESTINATION', }); + } + } + } - const temp_transformation_events = []; - for (const i in transformation_events) { - temp_transformation_events.push(transformation_events[i]); - vertices_list.push(transformation_events[i]); - } - - async.each(temp_transformation_events, (event, next) => { - db.addDocument('ot_vertices', event).then(() => { - updateImportNumber('ot_vertices', event, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing transformation events complete'); - }); + const { bizLocation } = event; + if (bizLocation) { + const bizLocationId = bizLocation.id; + eventEdges.push({ + _key: md5(`at_${senderId}_${eventId}_${bizLocationId}`), + _from: `ot_vertices/${eventKey}`, + _to: `${EDGE_KEY_TEMPLATE + bizLocationId}`, + edge_type: 'AT', + }); + } - for (const i in instance_of_edges) { - edges_list.push(instance_of_edges[i]); - } + if (event.readPoint) { + const locationReadPoint = event.readPoint.id; + eventEdges.push({ + _key: md5(`read_point_${senderId}_${eventId}_${locationReadPoint}`), + _from: `ot_vertices/${eventKey}`, + _to: `${EDGE_KEY_TEMPLATE + event.readPoint.id}`, + edge_type: 'AT', + }); + } - async.each(instance_of_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing instance_of edges complete'); - }); + if (event.inputEPCList) { + for (const inputEpc of arrayze(event.inputEPCList.epc)) { + const batchId = inputEpc; - for (const i in owned_by_edges) { - edges_list.push(owned_by_edges[i]); - } + eventEdges.push({ + _key: md5(`event_batch_${senderId}_${eventId}_${batchId}`), + _from: `ot_vertices/${eventKey}`, + _to: `${EDGE_KEY_TEMPLATE + batchId}`, + edge_type: 'INPUT_BATCH', + }); + } + } - async.each(owned_by_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing owned_by edges complete'); - }); + if (event.childEPCs) { + for (const inputEpc of arrayze(event.childEPCs)) { + const batchId = inputEpc.epc; - for (const i in at_edges) { - edges_list.push(at_edges[i]); - } + eventEdges.push({ + _key: md5(`event_batch_${senderId}_${eventId}_${batchId}`), + _from: `ot_vertices/${eventKey}`, + _to: `${EDGE_KEY_TEMPLATE + batchId}`, + edge_type: 'CHILD_BATCH', + }); + } + } - async.each(at_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing at_edges complete'); - }); + if (event.outputEPCList) { + for (const outputEpc of arrayze(event.outputEPCList.epc)) { + const batchId = outputEpc; + eventEdges.push({ + _key: md5(`event_batch_${senderId}_${eventId}_${batchId}`), + _from: `${EDGE_KEY_TEMPLATE + batchId}`, + _to: `ot_vertices/${eventKey}`, + edge_type: 'OUTPUT_BATCH', + }); + } + } - for (const i in read_point_edges) { - edges_list.push(read_point_edges[i]); - } - async.each(read_point_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing read_point edges complete'); - }); + if (event.parentID) { + const parentId = event.parentID; + // TODO: fetch from db. - for (const i in event_batch_edges) { - edges_list.push(event_batch_edges[i]); - } + // eventEdges.push({ + // _key: md5(`at_${senderId}_${eventId}_${biz_location}`), + // _from: `ot_vertices/${md5(`batch_${sender_id}_${parent_id}`)}`, + // _to: `ot_vertices/${md5(`event_${sender_id}_${event_id}`)}`, + // edge_type: 'PARENT_BATCH', + // }); + } + } - async.each(event_batch_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing event_batch edges complete'); - }); + const allVertices = + locationVertices + .concat(actorsVertices) + .concat(productVertices) + .concat(batchesVertices) + .concat(eventVertices); - for (const i in parent_batches_edges) { - edges_list.push(parent_batches_edges[i]); - } + const promises = allVertices.map(vertex => db.addDocument('ot_vertices', vertex)); + await Promise.all(promises); - async.each(parent_batches_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing parent_batches edges complete'); - }); + const classObjectEdges = []; - for (const i in child_batches_edges) { - edges_list.push(child_batches_edges[i]); - } + actorsVertices.forEach((vertex) => { + classObjectEdges.push({ + _key: md5(`is_${senderId}_${vertex.id}_${objectClassActorId}`), + _from: `ot_vertices/${vertex._key}`, + _to: `ot_vertices/${objectClassActorId}`, + edge_type: 'IS', + }); + }); + + productVertices.forEach((vertex) => { + classObjectEdges.push({ + _key: md5(`is_${senderId}_${vertex.id}_${objectClassProductId}`), + _from: `ot_vertices/${vertex._key}`, + _to: `ot_vertices/${objectClassProductId}`, + edge_type: 'IS', + }); + }); + + eventVertices.forEach((vertex) => { + vertex.data.categories.forEach(async (category) => { + const classKey = await db.getClassId(category); + classObjectEdges.push({ + _key: md5(`is_${senderId}_${vertex.id}_${classKey}`), + _from: `ot_vertices/${vertex._key}`, + _to: `ot_vertices/${classKey}`, + edge_type: 'IS', + }); + }); + }); - async.each(child_batches_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing child_batches edges complete'); - }); + const allEdges = locationEdges + .concat(eventEdges) + .concat(classObjectEdges); - for (const i in input_batches_edges) { - edges_list.push(input_batches_edges[i]); - } + for (const edge of allEdges) { + const to = edge._to; + const from = edge._from; - async.each(input_batches_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing input_batches edges complete'); - }); + if (to.startsWith(EDGE_KEY_TEMPLATE)) { + // eslint-disable-next-line + const vertex = await db.getVertexWithMaxVersion(to.substring(EDGE_KEY_TEMPLATE.length)); + edge._to = `ot_vertices/${vertex._key}`; + } + if (from.startsWith(EDGE_KEY_TEMPLATE)) { + // eslint-disable-next-line + const vertex = await db.getVertexWithMaxVersion(from.substring(EDGE_KEY_TEMPLATE.length)); + edge._from = `ot_vertices/${vertex._key}`; + } + } - for (const i in output_batches_edges) { - edges_list.push(output_batches_edges[i]); - } + await Promise.all(allEdges.map(edge => db.addDocument('ot_edges', edge))); - async.each(output_batches_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing output_batches edges complete'); - }); + await Promise.all(allVertices.map(vertex => db.updateDocumentImports('ot_vertices', vertex._key, importId))); - for (const i in business_location_edges) { - edges_list.push(business_location_edges[i]); - } + console.log('Done parsing and importing.'); + return { vertices: allVertices, edges: allEdges, import_id: importId }; +} - async.each(business_location_edges, (edge, next) => { - db.addDocument('ot_edges', edge).then(() => { - updateImportNumber('ot_edges', edge, import_id).then(() => { - next(); - }); - }); - }, () => { - console.log('Writing business_location edges complete'); - }); +async function parseGS1(gs1XmlFile) { + const gs1XmlFileBuffer = fs.readFileSync(gs1XmlFile); + const xsdFileBuffer = fs.readFileSync('./importers/EPCglobal-epcis-masterdata-1_2.xsd'); + const schema = xsd.parse(xsdFileBuffer.toString()); + const validationResult = schema.validate(gs1XmlFileBuffer.toString()); + if (validationResult !== null) { + throw Error(`Failed to validate schema. ${validationResult}`); + } - utilities.executeCallback( - callback, - { vertices: vertices_list, edges: edges_list, import_id }, - ); - } + const result = await new Promise((resolve, reject) => + parseString( + gs1XmlFileBuffer, + { explicitArray: false, mergeAttrs: true }, + /* eslint-disable consistent-return */ + async (err, json) => { + resolve(processXML(err, json)); }, - ); - }, + )); + + return result; +} + + +module.exports = () => ({ + parseGS1, }); + diff --git a/modules/importer.js b/modules/importer.js index 7dabe935df..38a566fc12 100644 --- a/modules/importer.js +++ b/modules/importer.js @@ -109,54 +109,50 @@ module.exports = () => { }); }, - importXMLgs1(ot_xml_document) { - return new Promise((resolve, reject) => { - gs1.parseGS1(ot_xml_document, (response) => { - log.info('[DC] Import complete'); + async importXMLgs1(ot_xml_document) { + try { + const result = await gs1.parseGS1(ot_xml_document); + log.info('[DC] Import complete'); - const result = response; - // eslint-disable-next-line prefer-destructuring - const vertices = result.vertices; - // eslint-disable-next-line prefer-destructuring - const edges = result.edges; - const data_id = result.import_id; + const { vertices } = result.vertices; + const { edges } = result.edges; + const data_id = result.import_id; - const leaves = []; - const hash_pairs = []; + const leaves = []; + const hash_pairs = []; - for (const i in vertices) { - // eslint-disable-next-line max-len - leaves.push(utilities.sha3(utilities.sortObject({ + for (const i in vertices) { + leaves.push(utilities.sha3(utilities.sortObject({ + identifiers: vertices[i].identifiers, + data: vertices[i].data, + }))); + hash_pairs.push({ + key: vertices[i]._key, + hash: utilities.sha3({ identifiers: vertices[i].identifiers, data: vertices[i].data, - }))); - // eslint-disable-next-line no-underscore-dangle - hash_pairs.push({ - key: vertices[i]._key, - hash: utilities.sha3({ - identifiers: vertices[i].identifiers, - data: vertices[i].data, - }), - }); // eslint-disable-line max-len - } - - const tree = new Mtree(hash_pairs); - const root_hash = utilities.sha3(tree.root()); - - log.info(`Import id: ${data_id}`); - log.info(`Import hash: ${root_hash}`); - resolve({ - data_id, - root_hash, - total_documents: hash_pairs.length, - vertices, - edges, + }), }); - }); - }); - }, + } + + const tree = new Mtree(hash_pairs); + const root_hash = utilities.sha3(tree.root()); + log.info(`Import id: ${data_id}`); + log.info(`Import hash: ${root_hash}`); + return { + data_id, + root_hash, + total_documents: hash_pairs.length, + vertices, + edges, + }; + } catch (error) { + log.error(`Failed to parse XML. Error ${error}.`); + return null; + } + }, }; return importer; diff --git a/ot-node.js b/ot-node.js index 40972a85c5..e4bfd3c833 100644 --- a/ot-node.js +++ b/ot-node.js @@ -283,6 +283,7 @@ class OTNode { const queryObject = { filepath: input_file, contact: req.contact, + response: res, }; globalEmitter.emit('gs1-import-request', queryObject); diff --git a/package.json b/package.json index 9741d5e02c..54f8902527 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail-node-0.6a", - "version": "0.6.0", + "version": "0.7.0", "description": "OriginTrail node", "main": ".eslintrc.js", "scripts": { diff --git a/test/modules/graphstorage.test.js b/test/modules/graphstorage.test.js index a52cb64048..f5be38b915 100644 --- a/test/modules/graphstorage.test.js +++ b/test/modules/graphstorage.test.js @@ -89,23 +89,6 @@ describe('GraphStorage module', () => { }); - it('attempt to create doc Collection on non existing db should fail', async () => { - try { - await myInvalidGraphStorage.createCollection(documentCollectionName); - } catch (error) { - assert.isTrue(error.toString().indexOf('Error: Not connected to graph database') >= 0); - } - }); - - it('attempt to create edge Collection on non existing db should fail', async () => { - try { - await myInvalidGraphStorage.createEdgeCollection(edgeCollectionName); - } catch (error) { - assert.isTrue(error.toString().indexOf('Error: Not connected to graph database') >= 0); - } - }); - - it('attempt to updateDocumentImports on non existing db should fail', async () => { try { await myInvalidGraphStorage.updateDocumentImports( @@ -118,35 +101,6 @@ describe('GraphStorage module', () => { } }); - it('.createCollection() should create Document Collection', async () => { - // first time creating Document Collection - await myGraphStorage.createCollection(documentCollectionName).then((response) => { - assert.equal(response, 'Collection created'); - }); - const myCollection = myGraphStorage.db.db.collection(documentCollectionName); - const data = await myCollection.get(); - assert.equal(data.code, 200); - assert.isFalse(data.isSystem); - assert.equal(data.name, documentCollectionName); - const info = await myGraphStorage.db.db.listCollections(); - assert.equal(info.length, 1); - }); - - it('.createEdgeCollection() should create Edge Collection', async () => { - // first time creating Edge Collection - await myGraphStorage.createEdgeCollection(edgeCollectionName).then((response) => { - assert.equal(response, 'Edge collection created'); - }); - - const myCollection = myGraphStorage.db.db.collection(edgeCollectionName); - const data = await myCollection.get(); - assert.equal(data.code, 200); - assert.isFalse(data.isSystem); - assert.equal(data.name, edgeCollectionName); - const info = await myGraphStorage.db.db.listCollections(); - assert.equal(info.length, 2); - }); - it('.addVertex() should save vertex in Document Collection', () => { myGraphStorage.addDocument(documentCollectionName, vertexOne).then((response) => { assert.containsAllKeys(response, ['_id', '_key', '_rev']); diff --git a/test/modules/neo4j.test.js b/test/modules/neo4j.test.js index 764cef1593..f9d7c4e6a2 100644 --- a/test/modules/neo4j.test.js +++ b/test/modules/neo4j.test.js @@ -129,9 +129,10 @@ describe('Neo4j module ', async () => { assert.equal(response, 3); }); - it('getVertexKeyWithMaxVersion', async () => { - const response = await testDb.getVertexKeyWithMaxVersion(vertexOne.identifiers.uid); + it('getVertexWithMaxVersion', async () => { + const response = await testDb.getVertexWithMaxVersion(vertexOne.identifiers.uid); console.log(response); + assert.deepEqual(response, vertexOneV3); }); it('getVerticesByImportId', async () => {