From 99cd111f809e162fb628869a27104866a77b6935 Mon Sep 17 00:00:00 2001 From: UniMa007 <35450966+UniMa007@users.noreply.github.com> Date: Thu, 5 May 2022 09:47:14 +0200 Subject: [PATCH] WIP: V6/feature/sparql optimization (#1819) * Initial add of sparqlquery-service.js * Add Integration Test Framework Mocha+Chai * Add basic testcase for sparqlquery-service.js * Add communica engine and axios; Implement HealthCheck * Implement Integration test to initialize SparqlService; Add Healtcheck * Add chai-as-promised for async testing * Implement Find Assertions By Keyword; Add parameter Validation * Add Testcase for findAssertsionByKeyword and helper funcions for param validatio * Implement findAssetsByKeyword; Refactor filterParameters to own method * Add Testcases for FindAssertionsByKeyword; CheckParameter and FindAssetsByKeyword * Implement construct, resolve method, transformBlankNodes; Implement executeQuery to be made generic * Add testcase for SPARQL Service, to be improved * Add testcase for SPARQL Service for testing insert * Upgrade SPARQL Comunica Lib to enable insertion * Add Insertion Logic, implement ask method; Minor refactoring * Add testcase for SPARQL Service for testing insert * Add latest comunica and chai plugin * Add missing functionality; Implement feedback from reviewers; * Adjust testcases to new endpoint and error handling * Add testconfig for sparqltests; Add Sparqlendpoint URL in config * Use endpoint url from config for testcases * Make functionality testcases generic integration tests. * Add own property for sparql update endpoint; According to SPARQL Spec update and retrieve endpoints can differ * Add TODO for missing logic in findAssertions; Add failing testcase * Fix mapping for findAssertions; Change to blazegraph impl. Co-authored-by: angrymob Co-authored-by: Name <> Co-authored-by: zeroxbt <89495162+zeroxbt@users.noreply.github.com> --- .origintrail_noderc.tests | 27 ++- external/sparqlquery-service.js | 219 ++++++++++++++++++ package.json | 2 + test/unit/sparlql-query-service.test.js | 289 ++++++++++++++++++++++++ 4 files changed, 535 insertions(+), 2 deletions(-) create mode 100644 external/sparqlquery-service.js create mode 100644 test/unit/sparlql-query-service.test.js diff --git a/.origintrail_noderc.tests b/.origintrail_noderc.tests index 51fc85c665..dbd6cf5f03 100644 --- a/.origintrail_noderc.tests +++ b/.origintrail_noderc.tests @@ -1,4 +1,27 @@ { -"network": { -} + "blockchain":[ + { + "blockchainTitle": "Polygon", + "networkId": "polygon::testnet", + "rpcEndpoints": ["https://rpc-mumbai.maticvigil.com/"], + "publicKey": "...", + "privateKey": "..." + } + ], + "graphDatabase": { + "username": "admin", + "password": "", + "implementation": "Blazegraph", + "url": "http://localhost:9999/blazegraph", + "sparqlEndpoint": "http://localhost:9999/blazegraph/namespace/kb/sparql", + "sparqlEndpointUpdate": "http://localhost:9999/blazegraph/namespace/kb/sparql" + }, + "logLevel": "trace", + "rpcPort": 8900, + "network": { + }, + "ipWhitelist": [ + "::1", + "127.0.0.1" + ] } diff --git a/external/sparqlquery-service.js b/external/sparqlquery-service.js new file mode 100644 index 0000000000..ed7b930b6a --- /dev/null +++ b/external/sparqlquery-service.js @@ -0,0 +1,219 @@ + +const constants = require('../modules/constants'); +const Engine = require('@comunica/query-sparql').QueryEngine; + +class SparqlqueryService { + constructor(config) { + this.config = config; + } + + async initialize(logger) { + this.logger = logger; + this.logger.info('Sparql Query module initialized successfully'); + this.queryEngine = new Engine(); + this.filtertype = { + KEYWORD: 'keyword', + KEYWORDPREFIX: 'keywordPrefix', + TYPES: 'types', + ISSUERS: 'issuers', + }; + this.context = { + sources: [{ + type: 'sparql', + value: `${this.config.sparqlEndpoint}`, + }], + baseIRI: 'http://schema.org/', + destination: { + type: 'sparql', + value: `${this.config.sparqlEndpointUpdate}`, + }, + log: this.logger, + }; + } + + async insert(triples, rootHash) { + const askQuery = `ASK WHERE { GRAPH <${rootHash}> { ?s ?p ?o } }`; + const exists = await this.ask(askQuery); + const insertion = ` + PREFIX schema: + INSERT DATA + { GRAPH <${rootHash}> + { ${triples} + } + }`; + if (!exists) { + await this.queryEngine.queryVoid(insertion, this.context); + return true; + } + } + + async construct(query) { + const result = await this.executeQuery(query); + return result; + } + + async ask(query) { + const result = await this.queryEngine.queryBoolean(query, this.context); + return result; + } + + async resolve(uri) { + this.logger.info('to be implemented by subclass'); + } + + async assertionsByAsset(uri) { + const query = `PREFIX schema: + SELECT ?assertionId ?issuer ?timestamp + WHERE { + ?assertionId schema:hasUALs "${uri}" ; + schema:hasTimestamp ?timestamp ; + schema:hasIssuer ?issuer . + } + ORDER BY DESC(?timestamp)`; + const result = await this.execute(query); + + return result; + } + + async findAssertions(nquads) { + const query = `SELECT ?g + WHERE { + GRAPH ?g { + ${nquads} + } + }`; + let graph = await this.execute(query); + graph = graph.map((x) => x.get('g') + .value + .replace(`${constants.DID_PREFIX}:`, '')); + if (graph.length && graph[0] === 'http://www.bigdata.com/rdf#nullGraph') { + return []; + } + return graph; + } + + async findAssertionsByKeyword(query, options, localQuery) { + if (options.prefix && !(typeof options.prefix === 'boolean')) { + this.logger.error(`Failed FindassertionsByKeyword: ${options.prefix} is not a boolean`); + throw new Error('Prefix is not an boolean'); + } + if (localQuery && !(typeof localQuery === 'boolean')) { + this.logger.error(`Failed FindassertionsByKeyword: ${localQuery} is not a boolean`); + throw new Error('Localquery is not an boolean'); + } + let limitQuery = ''; + limitQuery = this.createLimitQuery(options); + + const publicVisibilityQuery = !localQuery ? ' ?assertionId schema:hasVisibility "public" .' : ''; + const filterQuery = options.prefix ? this.createFilterParameter(query, this.filtertype.KEYWORDPREFIX) : this.createFilterParameter(query, this.filtertype.KEYWORD); + + const sparqlQuery = `PREFIX schema: + SELECT distinct ?assertionId + WHERE { + ?assertionId schema:hasKeywords ?keyword . + ${publicVisibilityQuery} + ${filterQuery} + } + ${limitQuery}`; + + const result = await this.execute(sparqlQuery); + return result; + } + + async findAssetsByKeyword(query, options, localQuery) { + if (options.prefix && !(typeof options.prefix === 'boolean')) { + this.logger.error(`Failed FindAssetsByKeyword: ${options.prefix} is not a boolean`); + // throw new Error('Prefix is not an boolean'); + } + if (localQuery && !(typeof localQuery === 'boolean')) { + this.logger.error(`Failed FindAssetsByKeyword: ${localQuery} is not a boolean`); + throw new Error('Localquery is not an boolean'); + } + query = this.cleanEscapeCharacter(query); + const limitQuery = this.createLimitQuery(options); + + const publicVisibilityQuery = !localQuery ? 'schema:hasVisibility "public" :' : ''; + const filterQuery = options.prefix ? this.createFilterParameter(query, this.filtertype.KEYWORDPREFIX) : this.createFilterParameter(query, this.filtertype.KEYWORD); + const issuerFilter = options.issuers ? this.createFilterParameter(options.issuers, this.filtertype.ISSUERS) : ''; + const typesFilter = options.types ? this.createFilterParameter(options.types, this.filtertype.TYPES) : ''; + + const sparqlQuery = `PREFIX schema: + SELECT ?assertionId ?assetId + WHERE { + ?assertionId schema:hasTimestamp ?latestTimestamp ; + ${publicVisibilityQuery} + schema:hasUALs ?assetId . + { + SELECT ?assetId (MAX(?timestamp) AS ?latestTimestamp) + WHERE { + ?assertionId schema:hasKeywords ?keyword ; + schema:hasIssuer ?issuer ; + schema:hasType ?type ; + schema:hasTimestamp ?timestamp ; + schema:hasUALs ?assetId . + ${filterQuery} + ${issuerFilter} + ${typesFilter} + } + GROUP BY ?assetId + ${limitQuery} + } + }`; + const result = await this.execute(sparqlQuery); + return result; + } + + async healthCheck() { + return true; + } + + async executeQuery(query) { + const test = await this.queryEngine.queryQuads(query, this.context); + return test.toArray(); + } + + async execute(query) { + const test = await this.queryEngine.queryBindings(query, this.context); + return test.toArray(); + } + + cleanEscapeCharacter(query) { + return query.replace(/['|[\]\\]/g, '\\$&'); + } + + createFilterParameter(queryParameter, type) { + queryParameter = this.cleanEscapeCharacter(queryParameter); + + switch (type) { + case this.filtertype.KEYWORD: + return `FILTER (lcase(?keyword) = '${queryParameter}')`; + case this.filtertype.KEYWORDPREFIX: + return `FILTER contains(lcase(?keyword),'${queryParameter}')`; + case this.filtertype.ISSUERS: + return `FILTER (?issuer IN (${JSON.stringify(queryParameter) + .slice(1, -1)}))`; + case this.filtertype.TYPES: + return `FILTER (?type IN (${JSON.stringify(queryParameter) + .slice(1, -1)}))`; + default: + return ''; + } + } + + createLimitQuery(options) { + if (!options.limit) { + return ''; + } + const queryLimit = Number(options.limit); + if (Number.isNaN(queryLimit) || !Number.isInteger(queryLimit)) { + this.logger.error(`Failed creating Limit query: ${options.limit} is not a number`); + throw new Error('Limit is not a number'); + } else if (Number.isInteger(options.limit) && options.limit < 0) { + this.logger.error(`Failed creating Limit query: ${options.limit} is negative number`); + throw new Error('Limit is not a number'); + } + return `LIMIT ${queryLimit}`; + } +} + +module.exports = SparqlqueryService; diff --git a/package.json b/package.json index 32f019f7f7..6d20fa1243 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "@babel/eslint-parser": "^7.16.5", "@cucumber/cucumber": "^8.0.0-rc.2", "chai": "^4.3.6", + "chai-as-promised": "^7.1.1", "dkg-client": "6.0.0-beta.1.16", "eslint": "^7.32.0", "eslint-config-airbnb": "^18.2.1", @@ -56,6 +57,7 @@ "solc": "0.7.6" }, "dependencies": { + "@comunica/query-sparql": "^2.2.1", "app-root-path": "^3.0.0", "awilix": "^5.0.1", "axios": "^0.24.0", diff --git a/test/unit/sparlql-query-service.test.js b/test/unit/sparlql-query-service.test.js new file mode 100644 index 0000000000..a73ba0acf7 --- /dev/null +++ b/test/unit/sparlql-query-service.test.js @@ -0,0 +1,289 @@ +const { + describe, + it, + before, + after, +} = require('mocha'); +const chai = require('chai'); + +const { + assert, + expect, +} = chai; + +const Sparql = require('../../external/sparqlquery-service'); +const Logger = require('../../modules/logger/logger'); +const fs = require('fs'); + +let sparqlService = null; +let logger = null; +chai.use(require('chai-as-promised')); + +this.makeId = function (length) { + let result = ''; + const characters = 'abcdefghijklmnopqrstuvwxyz0123456789'; + const charactersLength = characters.length; + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * + charactersLength)); + } + return result; +}; + +describe('Sparql module', () => { + before('Initialize Logger', async () => { + logger = new Logger('trace', false); + }); + before('Init Sparql Module', async () => { + const configFile = JSON.parse(fs.readFileSync('.origintrail_noderc.tests')); + let config = configFile.graphDatabase; + assert.isNotNull(config.sparqlEndpoint); + assert.isNotEmpty(config.sparqlEndpoint); + assert.isNotNull(config.sparqlEndpointUpdate); + assert.isNotEmpty(config.sparqlEndpointUpdate); + sparqlService = new Sparql(config); + await sparqlService.initialize(logger); + }); + it('Check for cleanup', async () => { + // Success + expect(sparqlService.cleanEscapeCharacter('keywordabc')) + .to + .equal('keywordabc'); + // Fail + expect(sparqlService.cleanEscapeCharacter('keywordabc\'')) + .to + .equal('keywordabc\\\''); + }); + it('Check limit creation', async () => { + // Success + expect(() => sparqlService.createLimitQuery({ limit: 'abc' })) + .to + .throw(Error); + + expect(() => sparqlService.createLimitQuery({ limit: Math.random() })) + .to + .throw(Error); + + expect(sparqlService.createLimitQuery({})) + .to + .equal(''); + // var randomnumber = Math.floor(Math.random() * (maximum - minimum + 1)) + minimum; + // eslint-disable-next-line no-bitwise + const random = (Math.random() * (99999999 - 1 + 1)) << 0; + const negativeRandom = random * -1; + + expect(sparqlService.createLimitQuery({ limit: random })) + .to + .equal(`LIMIT ${random}`); + + expect(() => sparqlService.createLimitQuery({ limit: negativeRandom })) + .to + .throw(Error); + }); + + it('Check FindAssertionsByKeyword Errorhandling', async () => { + await expect(sparqlService.findAssertionsByKeyword('abc', { limit: 'aaaa' }, false)) + .to + .be + .rejectedWith(Error); + + await expect(sparqlService.findAssertionsByKeyword('abc', { limit: '90' }, 'test')) + .to + .be + .rejectedWith(Error); + + await expect(sparqlService.findAssertionsByKeyword('abc', { + limit: '90', + prefix: 'test', + })) + .to + .be + .rejectedWith(Error); + }); + + it('Check FindAssertionsByKeyword functionality', async () => { + + let id = this.makeId(65); + const addTriple = await sparqlService.insert(` schema:hasKeywords "${id}" `, `did:dkg:${id}`); + // This can also be mocked if necessary + const test = await sparqlService.findAssertionsByKeyword(id, { + limit: 5, + prefix: true, + }, true); + expect(test) + .to + .be + .not + .empty; + + const testTwo = await sparqlService.findAssertionsByKeyword(id, { + limit: 5, + prefix: false, + }, true); + // eslint-disable-next-line no-unused-expressions + expect(testTwo) + .to + .be + .not + .empty; + }) + .timeout(600000); + + it('Check createFilterParameter', async () => { + expect(sparqlService.createFilterParameter('', '')) + .to + .equal(''); + + expect(sparqlService.createFilterParameter('\'', '')) + .to + .equal(''); + + expect(sparqlService.createFilterParameter('\'', sparqlService.filtertype.KEYWORD)) + .to + .equal('FILTER (lcase(?keyword) = \'\\\'\')'); + + expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.KEYWORD)) + .to + .equal('FILTER (lcase(?keyword) = \'abcd\')'); + + expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.KEYWORDPREFIX)) + .to + .equal('FILTER contains(lcase(?keyword),\'abcd\')'); + + expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.TYPES)) + .to + .equal('FILTER (?type IN (abcd))'); + + expect(sparqlService.createFilterParameter('abcd', sparqlService.filtertype.ISSUERS)) + .to + .equal('FILTER (?issuer IN (abcd))'); + }); + it('Check FindAssetsByKeyword functionality', async () => { + //Add new entry, so we can check if we find it really + + let id = this.makeId(65); + let triples = ` + schema:hasKeywords "${id}" . + schema:hasTimestamp "2022-04-18T06:48:05.123Z" . + schema:hasUALs "${id}" . + schema:hasIssuer "${id}" . + schema:hasType "${id}" . + `; + + const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); + expect(addTriple) + .to + .be + .true; + + const testContains = await sparqlService.findAssetsByKeyword(id.substring(1, 20), { + limit: 5, + prefix: true, + }, true); + // eslint-disable-next-line no-unused-expressions + expect(testContains) + .to + .be + .not + .empty; + + const testExact = await sparqlService.findAssetsByKeyword(id, { + limit: 5, + prefix: true, + }, true); + // eslint-disable-next-line no-unused-expressions + expect(testExact) + .to + .be + .not + .empty; + }) + .timeout(600000); + it('Check resolve functionality', async () => { + // This can also be mocked if necessary + const test = await sparqlService.resolve('0e62550721611b96321c7459e7790498240431025e46fce9cd99f2ea9763ffb0'); + // eslint-disable-next-line no-unused-expressions + expect(test) + .to + .be + .not + .null; + }) + .timeout(600000); + it('Check insert functionality', async () => { + // This can also be mocked if necessary + + let id = this.makeId(65); + const test = await sparqlService.insert(` schema:hasKeywords "${id}" `, `did:dkg:${id}`); + + // eslint-disable-next-line no-unused-expressions + expect(test) + .to + .be + .true; + }) + .timeout(600000); + + it('Check assertions By Asset functionality', async () => { + // This can also be mocked if necessary + + let id = this.makeId(65); + let triples = ` + schema:hasKeywords "${id}" . + schema:hasTimestamp "2022-04-18T06:48:05.123Z" . + schema:hasUALs "${id}" . + schema:hasIssuer "${id}" . + schema:hasType "${id}" . + `; + + const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); + expect(addTriple) + .to + .be + .true; + + const testExact = await sparqlService.assertionsByAsset(id, { + limit: 5, + prefix: true, + }, true); + // eslint-disable-next-line no-unused-expressions + expect(testExact) + .to + .be + .not + .empty; + + }) + .timeout(600000); + + it('Check find Assertions functionality', async () => { + // This can also be mocked if necessary + + let id = this.makeId(65); + let triples = ` + schema:hasKeywords "${id}" . + schema:hasTimestamp "2022-04-18T06:48:05.123Z" . + schema:hasUALs "${id}" . + schema:hasIssuer "${id}" . + schema:hasType "${id}" . + `; + + const addTriple = await sparqlService.insert(triples, `did:dkg:${id}`); + expect(addTriple) + .to + .be + .true; + + const testExact = await sparqlService.findAssertions(triples); + // eslint-disable-next-line no-unused-expressions + expect(testExact) + .to + .be + .not + .empty; + + }) + .timeout(600000); +}); + +