From b5b2c94d81c0be630ea4793a85f20bac3c12c54f Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Thu, 25 Apr 2024 15:41:26 +0100 Subject: [PATCH 01/15] issue-98 - adding optional option to create test report in the set location --- cli/package-lock.json | 58 ++++++++++++++++ cli/package.json | 2 + .../junit-report/junit-report.spec.ts | 66 +++++++++++++++++++ .../validate/junit-report/junit.report.ts | 54 +++++++++++++++ cli/src/commands/validate/validate.ts | 14 +++- cli/src/index.ts | 6 +- 6 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 cli/src/commands/validate/junit-report/junit-report.spec.ts create mode 100644 cli/src/commands/validate/junit-report/junit.report.ts diff --git a/cli/package-lock.json b/cli/package-lock.json index 4324af8c..9a68bb82 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -17,6 +17,7 @@ "graphviz-cli": "^2.0.0", "json-pointer": "^0.6.2", "lodash": "^4.17.21", + "junit-report-builder": "^3.2.1", "mkdirp": "^3.0.1", "ts-graphviz": "^2.1.1", "tsconfig-paths": "^4.2.0", @@ -30,6 +31,7 @@ "@types/jest": "^29.5.12", "@types/json-pointer": "^1.0.34", "@types/lodash": "^4.17.0", + "@types/junit-report-builder": "^3.0.2", "@types/node": "^20.11.30", "@typescript-eslint/eslint-plugin": "^6.21.0", "eslint": "^8.57.0", @@ -1985,6 +1987,10 @@ "version": "4.17.0", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==", + "node_modules/@types/junit-report-builder": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/junit-report-builder/-/junit-report-builder-3.0.2.tgz", + "integrity": "sha512-R5M+SYhMbwBeQcNXYWNCZkl09vkVfAtcPIaCGdzIkkbeaTrVbGQ7HVgi4s+EmM/M1K4ZuWQH0jGcvMvNePfxYA==", "dev": true }, "node_modules/@types/node": { @@ -3030,6 +3036,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/date-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-4.0.3.tgz", + "integrity": "sha512-7P3FyqDcfeznLZp2b+OMitV9Sz2lUnsT87WaTat9nVwqsBkTzPG3lPLNwW3en6F4pHUiWzr6vb8CLhjdK9bcxQ==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/debug": { "version": "4.3.4", "license": "MIT", @@ -5434,6 +5448,42 @@ "node": ">=0.10.0" } }, + "node_modules/junit-report-builder": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/junit-report-builder/-/junit-report-builder-3.2.1.tgz", + "integrity": "sha512-IMCp5XyDQ4YESDE4Za7im3buM0/7cMnRfe17k2X8B05FnUl9vqnaliX6cgOEmPIeWKfJrEe/gANRq/XgqttCqQ==", + "dependencies": { + "date-format": "4.0.3", + "lodash": "^4.17.21", + "make-dir": "^3.1.0", + "xmlbuilder": "^15.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/junit-report-builder/node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/junit-report-builder/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/keyv": { "version": "4.5.4", "dev": true, @@ -7238,6 +7288,14 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/xmlbuilder": { + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", + "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==", + "engines": { + "node": ">=8.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "license": "ISC", diff --git a/cli/package.json b/cli/package.json index c968be39..df8e51a4 100644 --- a/cli/package.json +++ b/cli/package.json @@ -27,6 +27,7 @@ "graphviz-cli": "^2.0.0", "json-pointer": "^0.6.2", "lodash": "^4.17.21", + "junit-report-builder": "^3.2.1", "mkdirp": "^3.0.1", "ts-graphviz": "^2.1.1", "tsconfig-paths": "^4.2.0", @@ -37,6 +38,7 @@ "@types/jest": "^29.5.12", "@types/json-pointer": "^1.0.34", "@types/lodash": "^4.17.0", + "@types/junit-report-builder": "^3.0.2", "@types/node": "^20.11.30", "@typescript-eslint/eslint-plugin": "^6.21.0", "eslint": "^8.57.0", diff --git a/cli/src/commands/validate/junit-report/junit-report.spec.ts b/cli/src/commands/validate/junit-report/junit-report.spec.ts new file mode 100644 index 00000000..e0328640 --- /dev/null +++ b/cli/src/commands/validate/junit-report/junit-report.spec.ts @@ -0,0 +1,66 @@ +import { ValidationOutput } from '../validation.output'; +import createJUnitReport from './junit.report'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +const jsonSchemaValidationOutput: ValidationOutput[] = [ + new ValidationOutput( + 'json-schema', + 'error', + 'must be integer', + '/path/to/node', + '#/node' + ) +]; + +const spectralValidationOutput: ValidationOutput[] = [ + new ValidationOutput( + 'no-empty-properties', + 'error', + 'Must not contain string properties set to the empty string or numerical properties set to zero', + '/relationships/0/relationship-type/connects/destination/interface' + ), + new ValidationOutput( + 'no-placeholder-properties-numerical', + 'warning', + 'Numerical placeholder (-1) detected in instantiated pattern.', + '/nodes/0/interfaces/0/port' + ) +]; + +describe('createJUnitReport', () => { + + it('should create a report with only JSON Schema Validations errors', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); + const jUnitReportLocation = tmpDir + '/test-report.xml'; + createJUnitReport(jsonSchemaValidationOutput, [], jUnitReportLocation); + expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should create a report with only Spectral issues', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); + const jUnitReportLocation = tmpDir + '/test-report.xml'; + createJUnitReport([], spectralValidationOutput, jUnitReportLocation); + expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should create a report with Spectral issues and JSON Schema errors', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); + const jUnitReportLocation = tmpDir + '/test-report.xml'; + createJUnitReport(jsonSchemaValidationOutput, spectralValidationOutput, jUnitReportLocation); + expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should create a report with no Spectral issues and no JSON Schema errors', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); + const jUnitReportLocation = tmpDir + '/test-report.xml'; + createJUnitReport([], [], jUnitReportLocation); + expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); + fs.rmSync(tmpDir, {recursive: true}); + }); + +}); diff --git a/cli/src/commands/validate/junit-report/junit.report.ts b/cli/src/commands/validate/junit-report/junit.report.ts new file mode 100644 index 00000000..8f90fd85 --- /dev/null +++ b/cli/src/commands/validate/junit-report/junit.report.ts @@ -0,0 +1,54 @@ +import junitReportBuilder from 'junit-report-builder'; +import { ValidationOutput } from '../validation.output'; + +export default function createJUnitReport(jsonSchemaValidationOutput: ValidationOutput[], runSpectralValidations: ValidationOutput[], location: string){ + + const suite = junitReportBuilder + .testSuite() + .name('JSON Schema Validation'); + + if (jsonSchemaValidationOutput.length <= 0) { + + suite.testCase() + .name('JSON Schema Validation succeeded'); + + } else { + + jsonSchemaValidationOutput.forEach(jsonSchemaError => { + suite.testCase() + .name(jsonSchemaError.message) + .failure(); + }); + + } + + const spectralSuite = junitReportBuilder + .testSuite() + .name('Spectral Suite'); + + if (runSpectralValidations.length <= 0) { + + spectralSuite + .testCase() + .name('Spectral Validation'); + + } else { + + runSpectralValidations.forEach(spectralIssue => { + if(spectralIssue.severity == 'error'){ + spectralSuite.testCase() + .name(spectralIssue.message) + .failure(); + }else{ + spectralSuite.testCase() + .name(spectralIssue.message); + } + }); + + } + + junitReportBuilder.writeTo(location); + +} + + diff --git a/cli/src/commands/validate/validate.ts b/cli/src/commands/validate/validate.ts index a81cfbcc..4b6d50b4 100644 --- a/cli/src/commands/validate/validate.ts +++ b/cli/src/commands/validate/validate.ts @@ -8,10 +8,11 @@ import * as winston from 'winston'; import { initLogger } from '../helper.js'; import { ValidationOutput as ValidationOutput } from './validation.output.js'; import { SpectralResult } from './spectral.result.js'; +import createJUnitReport from './junit-report/junit.report.js'; let logger: winston.Logger; // defined later at startup -export default async function validate(jsonSchemaInstantiationLocation: string, jsonSchemaLocation: string, metaSchemaPath: string, debug: boolean = false) { +export default async function validate(jsonSchemaInstantiationLocation: string, jsonSchemaLocation: string, metaSchemaPath: string, debug: boolean = false, junitReportLocation?: string) { logger = initLogger(debug); let errors = false; let validations: ValidationOutput[] = []; @@ -32,11 +33,18 @@ export default async function validate(jsonSchemaInstantiationLocation: string, errors = spectralResult.errors; validations = validations.concat(spectralResult.spectralIssues); + let jsonSchemaValidations = []; if (!validateSchema(jsonSchemaInstantiation)) { logger.debug(`JSON Schema validation raw output: ${prettifyJson(validateSchema.errors)}`); errors = true; - validations = validations.concat(formatJsonSchemaOutput(validateSchema.errors)); - } + jsonSchemaValidations = formatJsonSchemaOutput(validateSchema.errors); + validations = validations.concat(jsonSchemaValidations); + } + logger.info(junitReportLocation); + if(junitReportLocation) { + logger.debug("Generating test report file"); + createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, 'test-report.xml'); + } if(errors){ logger.error(`The following issues have been found on the JSON Schema instantiation ${prettifyJson(validations)}`); diff --git a/cli/src/index.ts b/cli/src/index.ts index 4e11ae6a..ee375c54 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -42,7 +42,11 @@ program .requiredOption('-p, --pattern ', 'Path to the pattern file to use. May be a file path or a URL.') .requiredOption('-i, --instantiation ', 'Path to the pattern instantiation file to use. May be a file path or a URL.') .option('-m, --metaSchemasLocation ', 'The location of the directory of the meta schemas to be loaded', '../calm/draft/2024-03/meta') + .option('-tr, --test-report ', 'Path location at which to output the generated test report.') .option('-v, --verbose', 'Enable verbose logging.', false) - .action(async (options)=> await validate(options.instantiation, options.pattern, options.metaSchemasLocation, options.verbose)); + .action(async (options)=>{ + console.log(options) + await validate(options.instantiation, options.pattern, options.metaSchemasLocation, options.verbose, options.testReport) + } ); program.parse(process.argv); From 0f294949b3878c10ae38e37c8433e73849f4c659 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Thu, 25 Apr 2024 16:52:50 +0100 Subject: [PATCH 02/15] issue-98 - adding the successful case for the test report for the spectral validation --- cli/package-lock.json | 11 +++- cli/package.json | 2 + .../junit-report/junit-report.spec.ts | 10 +-- .../validate/junit-report/junit.report.ts | 61 +++++++++---------- cli/src/commands/validate/validate.ts | 22 +++++-- cli/src/index.ts | 7 +-- 6 files changed, 65 insertions(+), 48 deletions(-) diff --git a/cli/package-lock.json b/cli/package-lock.json index 9a68bb82..8c80af81 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -17,6 +17,7 @@ "graphviz-cli": "^2.0.0", "json-pointer": "^0.6.2", "lodash": "^4.17.21", + "js-yaml": "^4.1.0", "junit-report-builder": "^3.2.1", "mkdirp": "^3.0.1", "ts-graphviz": "^2.1.1", @@ -31,6 +32,7 @@ "@types/jest": "^29.5.12", "@types/json-pointer": "^1.0.34", "@types/lodash": "^4.17.0", + "@types/js-yaml": "^4.0.9", "@types/junit-report-builder": "^3.0.2", "@types/node": "^20.11.30", "@typescript-eslint/eslint-plugin": "^6.21.0", @@ -1972,6 +1974,10 @@ "version": "1.0.34", "resolved": "https://registry.npmjs.org/@types/json-pointer/-/json-pointer-1.0.34.tgz", "integrity": "sha512-JRnWcxzXSaLei98xgw1B7vAeBVOrkyw0+Rt9j1QoJrczE78OpHsyQC8GNbuhw+/2vxxDe58QvWnngS86CoIbRg==", + "node_modules/@types/js-yaml": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", "dev": true }, "node_modules/@types/json-schema": { @@ -2400,7 +2406,6 @@ }, "node_modules/argparse": { "version": "2.0.1", - "dev": true, "license": "Python-2.0" }, "node_modules/array-buffer-byte-length": { @@ -5351,8 +5356,8 @@ }, "node_modules/js-yaml": { "version": "4.1.0", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dependencies": { "argparse": "^2.0.1" }, diff --git a/cli/package.json b/cli/package.json index df8e51a4..91f3e5ae 100644 --- a/cli/package.json +++ b/cli/package.json @@ -27,6 +27,7 @@ "graphviz-cli": "^2.0.0", "json-pointer": "^0.6.2", "lodash": "^4.17.21", + "js-yaml": "^4.1.0", "junit-report-builder": "^3.2.1", "mkdirp": "^3.0.1", "ts-graphviz": "^2.1.1", @@ -38,6 +39,7 @@ "@types/jest": "^29.5.12", "@types/json-pointer": "^1.0.34", "@types/lodash": "^4.17.0", + "@types/js-yaml": "^4.0.9", "@types/junit-report-builder": "^3.0.2", "@types/node": "^20.11.30", "@typescript-eslint/eslint-plugin": "^6.21.0", diff --git a/cli/src/commands/validate/junit-report/junit-report.spec.ts b/cli/src/commands/validate/junit-report/junit-report.spec.ts index e0328640..e1cc9f8f 100644 --- a/cli/src/commands/validate/junit-report/junit-report.spec.ts +++ b/cli/src/commands/validate/junit-report/junit-report.spec.ts @@ -29,12 +29,14 @@ const spectralValidationOutput: ValidationOutput[] = [ ) ]; +const ruleset = []; + describe('createJUnitReport', () => { it('should create a report with only JSON Schema Validations errors', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); const jUnitReportLocation = tmpDir + '/test-report.xml'; - createJUnitReport(jsonSchemaValidationOutput, [], jUnitReportLocation); + createJUnitReport(jsonSchemaValidationOutput, [], ruleset, jUnitReportLocation); expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); fs.rmSync(tmpDir, {recursive: true}); }); @@ -42,7 +44,7 @@ describe('createJUnitReport', () => { it('should create a report with only Spectral issues', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); const jUnitReportLocation = tmpDir + '/test-report.xml'; - createJUnitReport([], spectralValidationOutput, jUnitReportLocation); + createJUnitReport([], spectralValidationOutput, ruleset, jUnitReportLocation); expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); fs.rmSync(tmpDir, {recursive: true}); }); @@ -50,7 +52,7 @@ describe('createJUnitReport', () => { it('should create a report with Spectral issues and JSON Schema errors', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); const jUnitReportLocation = tmpDir + '/test-report.xml'; - createJUnitReport(jsonSchemaValidationOutput, spectralValidationOutput, jUnitReportLocation); + createJUnitReport(jsonSchemaValidationOutput, spectralValidationOutput, ruleset, jUnitReportLocation); expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); fs.rmSync(tmpDir, {recursive: true}); }); @@ -58,7 +60,7 @@ describe('createJUnitReport', () => { it('should create a report with no Spectral issues and no JSON Schema errors', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); const jUnitReportLocation = tmpDir + '/test-report.xml'; - createJUnitReport([], [], jUnitReportLocation); + createJUnitReport([], [], ruleset, jUnitReportLocation); expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); fs.rmSync(tmpDir, {recursive: true}); }); diff --git a/cli/src/commands/validate/junit-report/junit.report.ts b/cli/src/commands/validate/junit-report/junit.report.ts index 8f90fd85..2f934b70 100644 --- a/cli/src/commands/validate/junit-report/junit.report.ts +++ b/cli/src/commands/validate/junit-report/junit.report.ts @@ -1,54 +1,53 @@ -import junitReportBuilder from 'junit-report-builder'; +import junitReportBuilder, { TestSuite } from 'junit-report-builder'; import { ValidationOutput } from '../validation.output'; -export default function createJUnitReport(jsonSchemaValidationOutput: ValidationOutput[], runSpectralValidations: ValidationOutput[], location: string){ - - const suite = junitReportBuilder - .testSuite() - .name('JSON Schema Validation'); +export default function createJUnitReport( + jsonSchemaValidationOutput: ValidationOutput[], + spectralValidationOutput: ValidationOutput[], + spectralRules: string[], + outputLocation: string +){ + + const jsonSchemaSuite = createTestSuite('JSON Schema Validation'); if (jsonSchemaValidationOutput.length <= 0) { - - suite.testCase() - .name('JSON Schema Validation succeeded'); - + createTestCase(jsonSchemaSuite, 'JSON Schema Validation succeeded'); } else { - jsonSchemaValidationOutput.forEach(jsonSchemaError => { - suite.testCase() + jsonSchemaSuite.testCase() .name(jsonSchemaError.message) .failure(); }); - } - const spectralSuite = junitReportBuilder - .testSuite() - .name('Spectral Suite'); - - if (runSpectralValidations.length <= 0) { - - spectralSuite - .testCase() - .name('Spectral Validation'); + const spectralSuite = createTestSuite('Spectral Suite'); + if (spectralValidationOutput.length <= 0) { + spectralRules.forEach(ruleName => createTestCase(spectralSuite,ruleName)); } else { - - runSpectralValidations.forEach(spectralIssue => { - if(spectralIssue.severity == 'error'){ + spectralRules.forEach(ruleName => { + console.log(ruleName); + if (spectralValidationOutput.filter(item => (item.code === ruleName) && item.severity === 'error').length > 0) { spectralSuite.testCase() - .name(spectralIssue.message) + .name(ruleName) .failure(); - }else{ - spectralSuite.testCase() - .name(spectralIssue.message); + } else { + createTestCase(spectralSuite, ruleName); } }); - } - junitReportBuilder.writeTo(location); + junitReportBuilder.writeTo(outputLocation); +} +function createTestSuite(testSuiteName: string){ + return junitReportBuilder + .testSuite() + .name(testSuiteName); } +function createTestCase(testSuite: TestSuite, testName: string){ + testSuite.testCase() + .name(testName); +} diff --git a/cli/src/commands/validate/validate.ts b/cli/src/commands/validate/validate.ts index 4b6d50b4..aefc124d 100644 --- a/cli/src/commands/validate/validate.ts +++ b/cli/src/commands/validate/validate.ts @@ -9,6 +9,7 @@ import { initLogger } from '../helper.js'; import { ValidationOutput as ValidationOutput } from './validation.output.js'; import { SpectralResult } from './spectral.result.js'; import createJUnitReport from './junit-report/junit.report.js'; +import yaml from 'js-yaml'; let logger: winston.Logger; // defined later at startup @@ -29,7 +30,9 @@ export default async function validate(jsonSchemaInstantiationLocation: string, const validateSchema = ajv.compile(jsonSchema); - const spectralResult: SpectralResult = await runSpectralValidations(jsonSchemaInstantiation, stripRefs(jsonSchema)); + const spectralRuleset = '../spectral/instantiation/validation-rules.yaml'; + const spectralResult: SpectralResult = await runSpectralValidations(jsonSchemaInstantiation, stripRefs(jsonSchema), spectralRuleset); + errors = spectralResult.errors; validations = validations.concat(spectralResult.spectralIssues); @@ -40,10 +43,12 @@ export default async function validate(jsonSchemaInstantiationLocation: string, jsonSchemaValidations = formatJsonSchemaOutput(validateSchema.errors); validations = validations.concat(jsonSchemaValidations); } - logger.info(junitReportLocation); + + if(junitReportLocation) { - logger.debug("Generating test report file"); - createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, 'test-report.xml'); + logger.debug('Generating test report file'); + const spectralRules = extractRulesFromSpectralRuleset(spectralRuleset); + createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, spectralRules, 'test-report.xml'); } if(errors){ @@ -93,13 +98,13 @@ function loadMetaSchemas(ajv: Ajv2020, metaSchemaLocation: string) { }); } -async function runSpectralValidations(jsonSchemaInstantiation: string, jsonSchema: string): Promise { +async function runSpectralValidations(jsonSchemaInstantiation: string, jsonSchema: string, spectralRuleset: string): Promise { let errors = false; let spectralIssues: ValidationOutput[] = []; const spectral = new Spectral(); - spectral.setRuleset(await getRuleset('../spectral/instantiation/validation-rules.yaml')); + spectral.setRuleset(await getRuleset(spectralRuleset)); let issues = await spectral.run(jsonSchemaInstantiation); spectral.setRuleset(await getRuleset('../spectral/pattern/validation-rules.yaml')); issues = issues.concat(await spectral.run(jsonSchema)); @@ -188,6 +193,11 @@ async function loadFileFromUrl(fileUrl: string) { return body; } +function extractRulesFromSpectralRuleset(spectralRuleset: string){ + const yamlData = yaml.load(readFileSync(spectralRuleset, 'utf-8')); + return Object.keys(yamlData['rules']); +} + function prettifyJson(json){ return JSON.stringify(json, null, 4); } diff --git a/cli/src/index.ts b/cli/src/index.ts index ee375c54..8673cf7f 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -44,9 +44,8 @@ program .option('-m, --metaSchemasLocation ', 'The location of the directory of the meta schemas to be loaded', '../calm/draft/2024-03/meta') .option('-tr, --test-report ', 'Path location at which to output the generated test report.') .option('-v, --verbose', 'Enable verbose logging.', false) - .action(async (options)=>{ - console.log(options) - await validate(options.instantiation, options.pattern, options.metaSchemasLocation, options.verbose, options.testReport) - } ); + .action(async (options) => + await validate(options.instantiation, options.pattern, options.metaSchemasLocation, options.verbose, options.testReport) + ); program.parse(process.argv); From 28920bcda92b8afaab2d29e62e4fd96cbbbac925 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Fri, 26 Apr 2024 14:18:12 +0100 Subject: [PATCH 03/15] issue-98 - improving tests around the junit report creation --- .../junit-report/junit-report.spec.ts | 108 +++++++++++++++--- .../validate/junit-report/junit.report.ts | 12 +- 2 files changed, 98 insertions(+), 22 deletions(-) diff --git a/cli/src/commands/validate/junit-report/junit-report.spec.ts b/cli/src/commands/validate/junit-report/junit-report.spec.ts index e1cc9f8f..bfb1f94e 100644 --- a/cli/src/commands/validate/junit-report/junit-report.spec.ts +++ b/cli/src/commands/validate/junit-report/junit-report.spec.ts @@ -29,39 +29,115 @@ const spectralValidationOutput: ValidationOutput[] = [ ) ]; -const ruleset = []; +const ruleset = ["rules-number-1", 'rule-number-2', 'no-placeholder-properties-numerical', 'no-empty-properties']; + describe('createJUnitReport', () => { + let jUnitReportLocation: string; + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); + jUnitReportLocation = tmpDir + '/test-report.xml'; + }); it('should create a report with only JSON Schema Validations errors', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); - const jUnitReportLocation = tmpDir + '/test-report.xml'; createJUnitReport(jsonSchemaValidationOutput, [], ruleset, jUnitReportLocation); - expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); - fs.rmSync(tmpDir, {recursive: true}); + + expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location + + const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); + + const expected = ` + + + + + + + + + + + + + `; + + expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')) }); it('should create a report with only Spectral issues', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); - const jUnitReportLocation = tmpDir + '/test-report.xml'; createJUnitReport([], spectralValidationOutput, ruleset, jUnitReportLocation); - expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); - fs.rmSync(tmpDir, {recursive: true}); + + expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location + + const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); + const expected = ` + + + + + + + + + + + + + `; + expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')); + }); it('should create a report with Spectral issues and JSON Schema errors', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); - const jUnitReportLocation = tmpDir + '/test-report.xml'; + createJUnitReport(jsonSchemaValidationOutput, spectralValidationOutput, ruleset, jUnitReportLocation); - expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); - fs.rmSync(tmpDir, {recursive: true}); + + expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location + + const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); + const expected = ` + + + + + + + + + + + + + + + `; + expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')); }); it('should create a report with no Spectral issues and no JSON Schema errors', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); - const jUnitReportLocation = tmpDir + '/test-report.xml'; createJUnitReport([], [], ruleset, jUnitReportLocation); - expect(fs.existsSync(tmpDir + '/test-report.xml')).toBe(true); + + expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location + + const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); + const expected = ` + + + + + + + + + + + `; + expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')); + }); + + afterEach(() => { fs.rmSync(tmpDir, {recursive: true}); }); diff --git a/cli/src/commands/validate/junit-report/junit.report.ts b/cli/src/commands/validate/junit-report/junit.report.ts index 2f934b70..b12e8ba7 100644 --- a/cli/src/commands/validate/junit-report/junit.report.ts +++ b/cli/src/commands/validate/junit-report/junit.report.ts @@ -7,8 +7,9 @@ export default function createJUnitReport( spectralRules: string[], outputLocation: string ){ + const builder = junitReportBuilder.newBuilder(); - const jsonSchemaSuite = createTestSuite('JSON Schema Validation'); + const jsonSchemaSuite = createTestSuite(builder, 'JSON Schema Validation'); if (jsonSchemaValidationOutput.length <= 0) { createTestCase(jsonSchemaSuite, 'JSON Schema Validation succeeded'); @@ -20,13 +21,12 @@ export default function createJUnitReport( }); } - const spectralSuite = createTestSuite('Spectral Suite'); + const spectralSuite = createTestSuite(builder, 'Spectral Suite'); if (spectralValidationOutput.length <= 0) { spectralRules.forEach(ruleName => createTestCase(spectralSuite,ruleName)); } else { spectralRules.forEach(ruleName => { - console.log(ruleName); if (spectralValidationOutput.filter(item => (item.code === ruleName) && item.severity === 'error').length > 0) { spectralSuite.testCase() .name(ruleName) @@ -37,11 +37,11 @@ export default function createJUnitReport( }); } - junitReportBuilder.writeTo(outputLocation); + builder.writeTo(outputLocation); } -function createTestSuite(testSuiteName: string){ - return junitReportBuilder +function createTestSuite(builder, testSuiteName: string){ + return builder .testSuite() .name(testSuiteName); } From 3df1a7bc8eee849ac9a8adcb741a218ddbd276b2 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Fri, 26 Apr 2024 16:16:22 +0100 Subject: [PATCH 04/15] issue-98 - adding prefix to spectral rules name to make them clearer --- cli/src/commands/validate/validate.ts | 28 +++++++++++++------- spectral/instantiation/validation-rules.yaml | 18 ++++++------- spectral/pattern/validation-rules.yaml | 18 ++++++------- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/cli/src/commands/validate/validate.ts b/cli/src/commands/validate/validate.ts index aefc124d..b40032c9 100644 --- a/cli/src/commands/validate/validate.ts +++ b/cli/src/commands/validate/validate.ts @@ -30,8 +30,9 @@ export default async function validate(jsonSchemaInstantiationLocation: string, const validateSchema = ajv.compile(jsonSchema); - const spectralRuleset = '../spectral/instantiation/validation-rules.yaml'; - const spectralResult: SpectralResult = await runSpectralValidations(jsonSchemaInstantiation, stripRefs(jsonSchema), spectralRuleset); + const spectralRulesetForInstantiation = '../spectral/instantiation/validation-rules.yaml'; + const spectralRulesetForPattern = '../spectral/pattern/validation-rules.yaml'; + const spectralResult: SpectralResult = await runSpectralValidations(jsonSchemaInstantiation, stripRefs(jsonSchema), spectralRulesetForInstantiation, spectralRulesetForPattern); errors = spectralResult.errors; validations = validations.concat(spectralResult.spectralIssues); @@ -44,10 +45,9 @@ export default async function validate(jsonSchemaInstantiationLocation: string, validations = validations.concat(jsonSchemaValidations); } - if(junitReportLocation) { logger.debug('Generating test report file'); - const spectralRules = extractRulesFromSpectralRuleset(spectralRuleset); + const spectralRules = extractRulesFromSpectralRulesets(spectralRulesetForInstantiation, spectralRulesetForPattern); createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, spectralRules, 'test-report.xml'); } @@ -98,15 +98,20 @@ function loadMetaSchemas(ajv: Ajv2020, metaSchemaLocation: string) { }); } -async function runSpectralValidations(jsonSchemaInstantiation: string, jsonSchema: string, spectralRuleset: string): Promise { +async function runSpectralValidations( + jsonSchemaInstantiation: string, + jsonSchema: string, + spectralRulesetForInstantiation: string, + spectralRulesetForPattern: string +): Promise { let errors = false; let spectralIssues: ValidationOutput[] = []; const spectral = new Spectral(); - spectral.setRuleset(await getRuleset(spectralRuleset)); + spectral.setRuleset(await getRuleset(spectralRulesetForInstantiation)); let issues = await spectral.run(jsonSchemaInstantiation); - spectral.setRuleset(await getRuleset('../spectral/pattern/validation-rules.yaml')); + spectral.setRuleset(await getRuleset(spectralRulesetForPattern)); issues = issues.concat(await spectral.run(jsonSchema)); if (issues && issues.length > 0) { @@ -193,8 +198,13 @@ async function loadFileFromUrl(fileUrl: string) { return body; } -function extractRulesFromSpectralRuleset(spectralRuleset: string){ - const yamlData = yaml.load(readFileSync(spectralRuleset, 'utf-8')); +function extractRulesFromSpectralRulesets(spectralRulesetForInstantiation: string, spectralRulesetForPattern: string): string[]{ + return getRulesFromRulesetFile(spectralRulesetForInstantiation) + .concat(getRulesFromRulesetFile(spectralRulesetForPattern)); +} + +function getRulesFromRulesetFile(rulesetFile: string){ + const yamlData = yaml.load(readFileSync(rulesetFile, 'utf-8')); return Object.keys(yamlData['rules']); } diff --git a/spectral/instantiation/validation-rules.yaml b/spectral/instantiation/validation-rules.yaml index 6b8135a8..2803636f 100644 --- a/spectral/instantiation/validation-rules.yaml +++ b/spectral/instantiation/validation-rules.yaml @@ -5,7 +5,7 @@ functions: - numerical-placeholder rules: - has-nodes-relationships: + instantiation-has-nodes-relationships: description: Has top level nodes and relationships message: Should have nodes and relationships as top level properties on the CALM document severity: error @@ -16,7 +16,7 @@ rules: - field: relationships function: truthy - no-empty-properties: + no-empty-properties-instantiation: description: Must not contain string properties set to the empty string or numerical properties set to zero message: All properties must be set to a nonempty, nonzero value. severity: error @@ -25,7 +25,7 @@ rules: function: truthy - no-placeholder-properties-numerical: + no-placeholder-properties-numerical-in-instantiation: description: Should not contain numerical placeholder properties set to -1 message: Numerical placeholder (-1) detected in instantiated pattern. severity: warn @@ -34,7 +34,7 @@ rules: function: numerical-placeholder - no-placeholder-properties-string: + no-placeholder-properties-string-in-instantiation: description: Should not contain placeholder values with pattern {{ PLACEHOLDER_NAME }} message: String placeholder detected in instantiated pattern. severity: warn @@ -44,7 +44,7 @@ rules: functionOptions: notMatch: "^{{\\s*[A-Z_]+\\s*}}$" - relationship-references-existing-nodes: + relationship-references-existing-nodes-in-instantiation: description: Relationships must reference existing nodes severity: error message: "{{error}}" @@ -56,7 +56,7 @@ rules: - field: container function: node-id-exists - connects-relationship-references-existing-nodes: + connects-relationship-references-existing-nodes-in-instantiation: description: Connects relationships must reference existing nodes severity: error message: "{{error}}" @@ -66,7 +66,7 @@ rules: - field: node function: node-id-exists - referenced-interfaces-defined: + referenced-interfaces-defined-in-instantiation: description: Referenced interfaces must be defined severity: error message: "{{error}}" @@ -74,7 +74,7 @@ rules: then: function: interface-id-exists - composition-relationships-reference-existing-nodes: + composition-relationships-reference-existing-nodes-in-instantiation: description: All nodes in a composition relationship must reference existing nodes severity: error message: "{{error}}" @@ -82,7 +82,7 @@ rules: then: function: node-id-exists - nodes-must-be-referenced: + instantiation-nodes-must-be-referenced: description: Nodes must be referenced by at least one relationship. severity: warn message: "{{error}}" diff --git a/spectral/pattern/validation-rules.yaml b/spectral/pattern/validation-rules.yaml index ae277cfa..c2af35c7 100644 --- a/spectral/pattern/validation-rules.yaml +++ b/spectral/pattern/validation-rules.yaml @@ -5,7 +5,7 @@ functions: - numerical-placeholder rules: - has-nodes-relationships: + pattern-has-nodes-relationships: description: Has top level nodes and relationships message: Should have nodes and relationships as top level properties on the CALM document severity: error @@ -16,7 +16,7 @@ rules: - field: relationships function: truthy - no-empty-properties: + no-empty-properties-in-pattern: description: Must not contain string properties set to the empty string or numerical properties set to zero message: All properties must be set to a nonempty, nonzero value. severity: error @@ -25,7 +25,7 @@ rules: function: truthy - no-placeholder-properties-numerical: + no-placeholder-properties-numerical-in-pattern: description: Should not contain numerical placeholder properties set to -1 message: Numerical placeholder (-1) detected in instantiated pattern. severity: warn @@ -34,7 +34,7 @@ rules: function: numerical-placeholder - no-placeholder-properties-string: + no-placeholder-properties-string-in-pattern: description: Should not contain placeholder values with pattern {{ PLACEHOLDER_NAME }} message: String placeholder detected in instantiated pattern. severity: warn @@ -44,7 +44,7 @@ rules: functionOptions: notMatch: "^{{\\s*[A-Z_]+\\s*}}$" - connects-relationship-references-existing-nodes: + connects-relationship-references-existing-nodes-in-pattern: description: Connect relationships with const properties must reference existing nodes severity: error message: "{{error}}" @@ -54,7 +54,7 @@ rules: function: node-id-exists - group-relationship-references-existing-nodes: + group-relationship-references-existing-nodes-in-pattern: description: All nodes referenced by a grouping relationship must reference existing nodes severity: error message: "{{error}}" @@ -63,7 +63,7 @@ rules: # only container and actor are in scope, since source/destination use interfaces now function: node-id-exists - relationship-references-existing-nodes: + relationship-references-existing-nodes-in-pattern: description: Relationships with const properties must reference existing nodes severity: error message: "{{error}}" @@ -76,7 +76,7 @@ rules: field: container - referenced-interfaces-defined: + referenced-interfaces-defined-in-pattern: description: Referenced interfaces must be defined severity: error message: "{{error}}" @@ -85,7 +85,7 @@ rules: # node uses the interface structure so has a sub object function: interface-id-exists - nodes-must-be-referenced: + pattern-nodes-must-be-referenced: description: Nodes must be referenced by at least one relationship severity: warn message: "{{error}}" From 0aec2234d23d3ab524e71cedc9662db1e8480993 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Fri, 26 Apr 2024 16:22:24 +0100 Subject: [PATCH 05/15] issue-98 - updating readme with new option for validate, updating test name on junit report for json schema --- cli/README.md | 1 + cli/src/commands/validate/junit-report/junit-report.spec.ts | 4 ++-- cli/src/commands/validate/junit-report/junit.report.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/README.md b/cli/README.md index bb1e74fa..7eb4c3cc 100644 --- a/cli/README.md +++ b/cli/README.md @@ -78,6 +78,7 @@ Options: -p, --pattern Path to the pattern file to use. May be a file path or a URL. -i, --instantiation Path to the pattern instantiation file to use. May be a file path or a URL. -m, --metaSchemasLocation The location of the directory of the meta schemas to be loaded (default: "../calm/draft/2024-03/meta") + -tr, --test-report Path location at which to output the generated test report. -v, --verbose Enable verbose logging. (default: false) -h, --help display help for command diff --git a/cli/src/commands/validate/junit-report/junit-report.spec.ts b/cli/src/commands/validate/junit-report/junit-report.spec.ts index bfb1f94e..403e040e 100644 --- a/cli/src/commands/validate/junit-report/junit-report.spec.ts +++ b/cli/src/commands/validate/junit-report/junit-report.spec.ts @@ -51,7 +51,7 @@ describe('createJUnitReport', () => { const expected = ` - + @@ -100,7 +100,7 @@ describe('createJUnitReport', () => { const expected = ` - + diff --git a/cli/src/commands/validate/junit-report/junit.report.ts b/cli/src/commands/validate/junit-report/junit.report.ts index b12e8ba7..693d46b3 100644 --- a/cli/src/commands/validate/junit-report/junit.report.ts +++ b/cli/src/commands/validate/junit-report/junit.report.ts @@ -16,7 +16,7 @@ export default function createJUnitReport( } else { jsonSchemaValidationOutput.forEach(jsonSchemaError => { jsonSchemaSuite.testCase() - .name(jsonSchemaError.message) + .name(`${jsonSchemaError.message} at ${jsonSchemaError.schemaPath}`) .failure(); }); } From f870636508958947ae543092ee56dfb9fff59e81 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Mon, 29 Apr 2024 10:02:26 +0100 Subject: [PATCH 06/15] issue-98 run linter --- cli/src/commands/validate/junit-report/junit-report.spec.ts | 4 ++-- cli/src/commands/validate/validate.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/commands/validate/junit-report/junit-report.spec.ts b/cli/src/commands/validate/junit-report/junit-report.spec.ts index 403e040e..1d332d1b 100644 --- a/cli/src/commands/validate/junit-report/junit-report.spec.ts +++ b/cli/src/commands/validate/junit-report/junit-report.spec.ts @@ -29,7 +29,7 @@ const spectralValidationOutput: ValidationOutput[] = [ ) ]; -const ruleset = ["rules-number-1", 'rule-number-2', 'no-placeholder-properties-numerical', 'no-empty-properties']; +const ruleset = ['rules-number-1', 'rule-number-2', 'no-placeholder-properties-numerical', 'no-empty-properties']; describe('createJUnitReport', () => { @@ -63,7 +63,7 @@ describe('createJUnitReport', () => { `; - expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')) + expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')); }); it('should create a report with only Spectral issues', async () => { diff --git a/cli/src/commands/validate/validate.ts b/cli/src/commands/validate/validate.ts index b40032c9..f6056bf8 100644 --- a/cli/src/commands/validate/validate.ts +++ b/cli/src/commands/validate/validate.ts @@ -200,7 +200,7 @@ async function loadFileFromUrl(fileUrl: string) { function extractRulesFromSpectralRulesets(spectralRulesetForInstantiation: string, spectralRulesetForPattern: string): string[]{ return getRulesFromRulesetFile(spectralRulesetForInstantiation) - .concat(getRulesFromRulesetFile(spectralRulesetForPattern)); + .concat(getRulesFromRulesetFile(spectralRulesetForPattern)); } function getRulesFromRulesetFile(rulesetFile: string){ From 6e265b48f1a1039b57ced1410048b8ed4dcea036 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Mon, 29 Apr 2024 10:46:17 +0100 Subject: [PATCH 07/15] issue-98 - improving spectral rules names --- spectral/instantiation/validation-rules.yaml | 6 +++--- spectral/pattern/validation-rules.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spectral/instantiation/validation-rules.yaml b/spectral/instantiation/validation-rules.yaml index 2803636f..50eec38b 100644 --- a/spectral/instantiation/validation-rules.yaml +++ b/spectral/instantiation/validation-rules.yaml @@ -16,7 +16,7 @@ rules: - field: relationships function: truthy - no-empty-properties-instantiation: + instantiation-has-no-empty-properties: description: Must not contain string properties set to the empty string or numerical properties set to zero message: All properties must be set to a nonempty, nonzero value. severity: error @@ -25,7 +25,7 @@ rules: function: truthy - no-placeholder-properties-numerical-in-instantiation: + instantiation-has-no-placeholder-properties-numerical: description: Should not contain numerical placeholder properties set to -1 message: Numerical placeholder (-1) detected in instantiated pattern. severity: warn @@ -34,7 +34,7 @@ rules: function: numerical-placeholder - no-placeholder-properties-string-in-instantiation: + instantiation-has-no-placeholder-properties-string: description: Should not contain placeholder values with pattern {{ PLACEHOLDER_NAME }} message: String placeholder detected in instantiated pattern. severity: warn diff --git a/spectral/pattern/validation-rules.yaml b/spectral/pattern/validation-rules.yaml index c2af35c7..75764ece 100644 --- a/spectral/pattern/validation-rules.yaml +++ b/spectral/pattern/validation-rules.yaml @@ -16,7 +16,7 @@ rules: - field: relationships function: truthy - no-empty-properties-in-pattern: + pattern-has-no-empty-properties: description: Must not contain string properties set to the empty string or numerical properties set to zero message: All properties must be set to a nonempty, nonzero value. severity: error @@ -25,7 +25,7 @@ rules: function: truthy - no-placeholder-properties-numerical-in-pattern: + pattern-has-no-placeholder-properties-numerical: description: Should not contain numerical placeholder properties set to -1 message: Numerical placeholder (-1) detected in instantiated pattern. severity: warn @@ -34,7 +34,7 @@ rules: function: numerical-placeholder - no-placeholder-properties-string-in-pattern: + pattern-has-no-placeholder-properties-string: description: Should not contain placeholder values with pattern {{ PLACEHOLDER_NAME }} message: String placeholder detected in instantiated pattern. severity: warn From e53c08709b8a71c33babed4eaa4d27fa41e05aa3 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Mon, 29 Apr 2024 11:09:22 +0100 Subject: [PATCH 08/15] issue-98 - adding check for xml extension and adding tests for creating test report --- cli/src/commands/validate/validate.spec.ts | 41 +++++++++++++++++++++- cli/src/commands/validate/validate.ts | 5 ++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/cli/src/commands/validate/validate.spec.ts b/cli/src/commands/validate/validate.spec.ts index 1b983ee6..a2ed2fb7 100644 --- a/cli/src/commands/validate/validate.spec.ts +++ b/cli/src/commands/validate/validate.spec.ts @@ -1,10 +1,11 @@ import fetchMock from 'fetch-mock'; import validate, { exportedForTesting } from './validate'; -import { readFileSync } from 'fs'; +import { readFileSync, mkdtempSync, existsSync, rmSync } from 'fs'; import path from 'path'; import { ISpectralDiagnostic } from '@stoplight/spectral-core'; import { ValidationOutput } from './validation.output'; import { ErrorObject } from 'ajv'; +import os from 'os'; const mockRunFunction = jest.fn(); @@ -182,6 +183,14 @@ describe('validate', () => { expect(mockExit).toHaveBeenCalledWith(1); }); + it('exits with error when the test report output location is not an xml file', async () => { + await expect(validate('test_fixtures/api-gateway-implementation.json', 'test_fixtures/api-gateway.json', metaSchemaLocation, debugDisabled, 'not-xml.json')) + .rejects + .toThrow(); + + expect(mockExit).toHaveBeenCalledWith(1); + }); + it('complete successfully when the pattern instantiation validates against the pattern json schema', async () => { const mockExit = jest.spyOn(process, 'exit') .mockImplementation((code) => { @@ -203,6 +212,36 @@ describe('validate', () => { fetchMock.restore(); }); + it('complete successfully when the pattern instantiation validates against the pattern json schema and the test report is created', async () => { + const mockExit = jest.spyOn(process, 'exit') + .mockImplementation((code) => { + if (code != 0) { + throw new Error(); + } + return undefined as never; + }); + + const apiGateway = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway.json'), 'utf8'); + fetchMock.mock('http://exist/api-gateway.json', apiGateway); + + const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8'); + fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); + + const tmpDir = mkdtempSync(path.join(os.tmpdir())); + const testReportLocation = tmpDir + '/test-report.xml'; + console.log(testReportLocation); + await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, testReportLocation); + + expect(existsSync(testReportLocation)).toBe(true); //check that the test report exists + + rmSync(tmpDir, {recursive: true}); //delete folder with the test report + + expect(mockExit).toHaveBeenCalledWith(0); + fetchMock.restore(); + + + }); + }); const { diff --git a/cli/src/commands/validate/validate.ts b/cli/src/commands/validate/validate.ts index f6056bf8..1a71dfa3 100644 --- a/cli/src/commands/validate/validate.ts +++ b/cli/src/commands/validate/validate.ts @@ -46,9 +46,12 @@ export default async function validate(jsonSchemaInstantiationLocation: string, } if(junitReportLocation) { + if (!junitReportLocation.endsWith('.xml')){ + throw new Error('The test report location must be an xml file'); + } logger.debug('Generating test report file'); const spectralRules = extractRulesFromSpectralRulesets(spectralRulesetForInstantiation, spectralRulesetForPattern); - createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, spectralRules, 'test-report.xml'); + createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, spectralRules, junitReportLocation); } if(errors){ From 907940b90548a91b511a758838a5fe671cffdfe4 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Mon, 29 Apr 2024 11:19:33 +0100 Subject: [PATCH 09/15] issue-98 updating package-lock.json --- cli/package-lock.json | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/cli/package-lock.json b/cli/package-lock.json index 8c80af81..d7f49588 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -15,10 +15,10 @@ "commander": "^12.0.0", "fetch-mock": "^9.11.0", "graphviz-cli": "^2.0.0", - "json-pointer": "^0.6.2", - "lodash": "^4.17.21", "js-yaml": "^4.1.0", + "json-pointer": "^0.6.2", "junit-report-builder": "^3.2.1", + "lodash": "^4.17.21", "mkdirp": "^3.0.1", "ts-graphviz": "^2.1.1", "tsconfig-paths": "^4.2.0", @@ -30,10 +30,10 @@ "devDependencies": { "@jest/globals": "^29.7.0", "@types/jest": "^29.5.12", - "@types/json-pointer": "^1.0.34", - "@types/lodash": "^4.17.0", "@types/js-yaml": "^4.0.9", + "@types/json-pointer": "^1.0.34", "@types/junit-report-builder": "^3.0.2", + "@types/lodash": "^4.17.0", "@types/node": "^20.11.30", "@typescript-eslint/eslint-plugin": "^6.21.0", "eslint": "^8.57.0", @@ -1970,16 +1970,18 @@ "pretty-format": "^29.0.0" } }, - "node_modules/@types/json-pointer": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/@types/json-pointer/-/json-pointer-1.0.34.tgz", - "integrity": "sha512-JRnWcxzXSaLei98xgw1B7vAeBVOrkyw0+Rt9j1QoJrczE78OpHsyQC8GNbuhw+/2vxxDe58QvWnngS86CoIbRg==", "node_modules/@types/js-yaml": { "version": "4.0.9", "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", "dev": true }, + "node_modules/@types/json-pointer": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/@types/json-pointer/-/json-pointer-1.0.34.tgz", + "integrity": "sha512-JRnWcxzXSaLei98xgw1B7vAeBVOrkyw0+Rt9j1QoJrczE78OpHsyQC8GNbuhw+/2vxxDe58QvWnngS86CoIbRg==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.15", "license": "MIT" @@ -1989,16 +1991,18 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/lodash": { - "version": "4.17.0", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", - "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==", "node_modules/@types/junit-report-builder": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/junit-report-builder/-/junit-report-builder-3.0.2.tgz", "integrity": "sha512-R5M+SYhMbwBeQcNXYWNCZkl09vkVfAtcPIaCGdzIkkbeaTrVbGQ7HVgi4s+EmM/M1K4ZuWQH0jGcvMvNePfxYA==", "dev": true }, + "node_modules/@types/lodash": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", + "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==", + "dev": true + }, "node_modules/@types/node": { "version": "20.11.30", "license": "MIT", @@ -5541,8 +5545,7 @@ }, "node_modules/lodash": { "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + "license": "MIT" }, "node_modules/lodash.isequal": { "version": "4.5.0", @@ -6837,9 +6840,8 @@ }, "node_modules/ts-jest": { "version": "29.1.2", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.2.tgz", - "integrity": "sha512-br6GJoH/WUX4pu7FbZXuWGKGNDuU7b8Uj77g/Sp7puZV6EXzuByl6JrECvm0MzVzSTkSHWTihsXt+5XYER5b+g==", "dev": true, + "license": "MIT", "dependencies": { "bs-logger": "0.x", "fast-json-stable-stringify": "2.x", From 689b58bbf2c1a6f561b7ade0645a5d2f34f8479b Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Mon, 29 Apr 2024 12:52:20 +0100 Subject: [PATCH 10/15] issue-98 renaming test file and removing console.log --- .../junit-report/{junit-report.spec.ts => junit.report.spec.ts} | 0 cli/src/commands/validate/validate.spec.ts | 1 - 2 files changed, 1 deletion(-) rename cli/src/commands/validate/junit-report/{junit-report.spec.ts => junit.report.spec.ts} (100%) diff --git a/cli/src/commands/validate/junit-report/junit-report.spec.ts b/cli/src/commands/validate/junit-report/junit.report.spec.ts similarity index 100% rename from cli/src/commands/validate/junit-report/junit-report.spec.ts rename to cli/src/commands/validate/junit-report/junit.report.spec.ts diff --git a/cli/src/commands/validate/validate.spec.ts b/cli/src/commands/validate/validate.spec.ts index a2ed2fb7..cf7ca56f 100644 --- a/cli/src/commands/validate/validate.spec.ts +++ b/cli/src/commands/validate/validate.spec.ts @@ -229,7 +229,6 @@ describe('validate', () => { const tmpDir = mkdtempSync(path.join(os.tmpdir())); const testReportLocation = tmpDir + '/test-report.xml'; - console.log(testReportLocation); await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, testReportLocation); expect(existsSync(testReportLocation)).toBe(true); //check that the test report exists From 759bd5da0edffc8ca5abe708848be2b1651dab5f Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Mon, 29 Apr 2024 13:05:21 +0100 Subject: [PATCH 11/15] issue-98 - using path.join to join directory location and file name --- cli/src/commands/validate/junit-report/junit.report.spec.ts | 2 +- cli/src/commands/validate/validate.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/commands/validate/junit-report/junit.report.spec.ts b/cli/src/commands/validate/junit-report/junit.report.spec.ts index 1d332d1b..1eb302cf 100644 --- a/cli/src/commands/validate/junit-report/junit.report.spec.ts +++ b/cli/src/commands/validate/junit-report/junit.report.spec.ts @@ -38,7 +38,7 @@ describe('createJUnitReport', () => { beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); - jUnitReportLocation = tmpDir + '/test-report.xml'; + jUnitReportLocation = path.join(tmpDir, 'test-report.xml'); }); it('should create a report with only JSON Schema Validations errors', async () => { diff --git a/cli/src/commands/validate/validate.spec.ts b/cli/src/commands/validate/validate.spec.ts index cf7ca56f..30e87b6b 100644 --- a/cli/src/commands/validate/validate.spec.ts +++ b/cli/src/commands/validate/validate.spec.ts @@ -228,7 +228,7 @@ describe('validate', () => { fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); const tmpDir = mkdtempSync(path.join(os.tmpdir())); - const testReportLocation = tmpDir + '/test-report.xml'; + const testReportLocation = path.join(tmpDir, 'test-report.xml'); await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, testReportLocation); expect(existsSync(testReportLocation)).toBe(true); //check that the test report exists From 41a4c2ca8e25cb760e54642ab59da6e19aadde18 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Mon, 29 Apr 2024 13:08:34 +0100 Subject: [PATCH 12/15] issue-98 - adding new folder to the folder structure to test junit report --- cli/src/commands/validate/junit-report/junit.report.spec.ts | 2 +- cli/src/commands/validate/validate.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/commands/validate/junit-report/junit.report.spec.ts b/cli/src/commands/validate/junit-report/junit.report.spec.ts index 1eb302cf..61fae67d 100644 --- a/cli/src/commands/validate/junit-report/junit.report.spec.ts +++ b/cli/src/commands/validate/junit-report/junit.report.spec.ts @@ -37,7 +37,7 @@ describe('createJUnitReport', () => { let tmpDir: string; beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join(os.tmpdir())); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'report')); jUnitReportLocation = path.join(tmpDir, 'test-report.xml'); }); diff --git a/cli/src/commands/validate/validate.spec.ts b/cli/src/commands/validate/validate.spec.ts index 30e87b6b..c4218719 100644 --- a/cli/src/commands/validate/validate.spec.ts +++ b/cli/src/commands/validate/validate.spec.ts @@ -227,7 +227,7 @@ describe('validate', () => { const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8'); fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); - const tmpDir = mkdtempSync(path.join(os.tmpdir())); + const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'report')); const testReportLocation = path.join(tmpDir, 'test-report.xml'); await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, testReportLocation); From 1ceb59f7f0cb020dd9ad34f45f35fbf8bca2d1fb Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Tue, 30 Apr 2024 10:58:50 +0100 Subject: [PATCH 13/15] issue-98 - addressing pr comments and creating new method for failing test case --- .../validate/junit-report/junit.report.ts | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/cli/src/commands/validate/junit-report/junit.report.ts b/cli/src/commands/validate/junit-report/junit.report.ts index 693d46b3..dfcfb10a 100644 --- a/cli/src/commands/validate/junit-report/junit.report.ts +++ b/cli/src/commands/validate/junit-report/junit.report.ts @@ -4,7 +4,7 @@ import { ValidationOutput } from '../validation.output'; export default function createJUnitReport( jsonSchemaValidationOutput: ValidationOutput[], spectralValidationOutput: ValidationOutput[], - spectralRules: string[], + spectralRulesName: string[], outputLocation: string ){ const builder = junitReportBuilder.newBuilder(); @@ -12,27 +12,24 @@ export default function createJUnitReport( const jsonSchemaSuite = createTestSuite(builder, 'JSON Schema Validation'); if (jsonSchemaValidationOutput.length <= 0) { - createTestCase(jsonSchemaSuite, 'JSON Schema Validation succeeded'); + createSucceedingTestCase(jsonSchemaSuite, 'JSON Schema Validation succeeded'); } else { jsonSchemaValidationOutput.forEach(jsonSchemaError => { - jsonSchemaSuite.testCase() - .name(`${jsonSchemaError.message} at ${jsonSchemaError.schemaPath}`) - .failure(); + const testName = `${jsonSchemaError.message} at ${jsonSchemaError.schemaPath}`; + createFailingTestCase(jsonSchemaSuite, testName); }); } const spectralSuite = createTestSuite(builder, 'Spectral Suite'); if (spectralValidationOutput.length <= 0) { - spectralRules.forEach(ruleName => createTestCase(spectralSuite,ruleName)); + spectralRulesName.forEach(ruleName => createSucceedingTestCase(spectralSuite,ruleName)); } else { - spectralRules.forEach(ruleName => { + spectralRulesName.forEach(ruleName => { if (spectralValidationOutput.filter(item => (item.code === ruleName) && item.severity === 'error').length > 0) { - spectralSuite.testCase() - .name(ruleName) - .failure(); + createFailingTestCase(spectralSuite, ruleName); } else { - createTestCase(spectralSuite, ruleName); + createSucceedingTestCase(spectralSuite, ruleName); } }); } @@ -46,8 +43,14 @@ function createTestSuite(builder, testSuiteName: string){ .name(testSuiteName); } -function createTestCase(testSuite: TestSuite, testName: string){ +function createSucceedingTestCase(testSuite: TestSuite, testName: string){ testSuite.testCase() .name(testName); } +function createFailingTestCase(testSuite: TestSuite, testName: string){ + testSuite.testCase() + .name(testName) + .failure(); +} + From 481f9d42a7bd6db00064d7116f985432a139defc Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Tue, 30 Apr 2024 16:50:02 +0100 Subject: [PATCH 14/15] issue-98 - adding format and output option for validate command --- cli/README.md | 3 +- .../junit-report/junit.report.spec.ts | 38 ++------- .../validate/junit-report/junit.report.ts | 7 +- cli/src/commands/validate/validate.spec.ts | 80 ++++++++++++++----- cli/src/commands/validate/validate.ts | 61 ++++++++++---- cli/src/index.ts | 9 ++- 6 files changed, 122 insertions(+), 76 deletions(-) diff --git a/cli/README.md b/cli/README.md index 7eb4c3cc..b4bbd05a 100644 --- a/cli/README.md +++ b/cli/README.md @@ -78,7 +78,8 @@ Options: -p, --pattern Path to the pattern file to use. May be a file path or a URL. -i, --instantiation Path to the pattern instantiation file to use. May be a file path or a URL. -m, --metaSchemasLocation The location of the directory of the meta schemas to be loaded (default: "../calm/draft/2024-03/meta") - -tr, --test-report Path location at which to output the generated test report. + -f, --format The format of the output (choices: "json", "junit", default: "json") + -o, --output Path location at which to output the generated file. -v, --verbose Enable verbose logging. (default: false) -h, --help display help for command diff --git a/cli/src/commands/validate/junit-report/junit.report.spec.ts b/cli/src/commands/validate/junit-report/junit.report.spec.ts index 61fae67d..411c98be 100644 --- a/cli/src/commands/validate/junit-report/junit.report.spec.ts +++ b/cli/src/commands/validate/junit-report/junit.report.spec.ts @@ -1,8 +1,5 @@ import { ValidationOutput } from '../validation.output'; import createJUnitReport from './junit.report'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; const jsonSchemaValidationOutput: ValidationOutput[] = [ new ValidationOutput( @@ -33,20 +30,8 @@ const ruleset = ['rules-number-1', 'rule-number-2', 'no-placeholder-properties-n describe('createJUnitReport', () => { - let jUnitReportLocation: string; - let tmpDir: string; - - beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'report')); - jUnitReportLocation = path.join(tmpDir, 'test-report.xml'); - }); - it('should create a report with only JSON Schema Validations errors', async () => { - createJUnitReport(jsonSchemaValidationOutput, [], ruleset, jUnitReportLocation); - - expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location - - const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); + const actual = createJUnitReport(jsonSchemaValidationOutput, [], ruleset); const expected = ` @@ -67,11 +52,8 @@ describe('createJUnitReport', () => { }); it('should create a report with only Spectral issues', async () => { - createJUnitReport([], spectralValidationOutput, ruleset, jUnitReportLocation); + const actual = createJUnitReport([], spectralValidationOutput, ruleset); - expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location - - const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); const expected = ` @@ -91,12 +73,8 @@ describe('createJUnitReport', () => { }); it('should create a report with Spectral issues and JSON Schema errors', async () => { + const actual = createJUnitReport(jsonSchemaValidationOutput, spectralValidationOutput, ruleset); - createJUnitReport(jsonSchemaValidationOutput, spectralValidationOutput, ruleset, jUnitReportLocation); - - expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location - - const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); const expected = ` @@ -117,11 +95,8 @@ describe('createJUnitReport', () => { }); it('should create a report with no Spectral issues and no JSON Schema errors', async () => { - createJUnitReport([], [], ruleset, jUnitReportLocation); - - expect(fs.existsSync(jUnitReportLocation)).toBe(true); //check if the file was created in the correct location + const actual = createJUnitReport([], [], ruleset); - const actual = fs.readFileSync(jUnitReportLocation, 'utf-8'); const expected = ` @@ -134,11 +109,8 @@ describe('createJUnitReport', () => { `; - expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')); - }); - afterEach(() => { - fs.rmSync(tmpDir, {recursive: true}); + expect(actual.replace(/\s/g, '')).toBe(expected.replace(/\s/g, '')); }); }); diff --git a/cli/src/commands/validate/junit-report/junit.report.ts b/cli/src/commands/validate/junit-report/junit.report.ts index dfcfb10a..1e6c664f 100644 --- a/cli/src/commands/validate/junit-report/junit.report.ts +++ b/cli/src/commands/validate/junit-report/junit.report.ts @@ -4,9 +4,8 @@ import { ValidationOutput } from '../validation.output'; export default function createJUnitReport( jsonSchemaValidationOutput: ValidationOutput[], spectralValidationOutput: ValidationOutput[], - spectralRulesName: string[], - outputLocation: string -){ + spectralRulesName: string[] +): string { const builder = junitReportBuilder.newBuilder(); const jsonSchemaSuite = createTestSuite(builder, 'JSON Schema Validation'); @@ -34,7 +33,7 @@ export default function createJUnitReport( }); } - builder.writeTo(outputLocation); + return builder.build(); } function createTestSuite(builder, testSuiteName: string){ diff --git a/cli/src/commands/validate/validate.spec.ts b/cli/src/commands/validate/validate.spec.ts index c4218719..cc17f0aa 100644 --- a/cli/src/commands/validate/validate.spec.ts +++ b/cli/src/commands/validate/validate.spec.ts @@ -36,6 +36,8 @@ jest.mock('../helper.js', () => { const metaSchemaLocation = 'test_fixtures/calm'; const debugDisabled = false; +const jsonFormat = 'json'; +const jUnitFormat = 'junit'; describe('validate', () => { let mockExit; @@ -52,7 +54,7 @@ describe('validate', () => { it('exits with error when the JSON Schema pattern cannot be found in the input path', async () => { - await expect(validate('../test_fixtures/api-gateway-implementation.json', 'thisFolderDoesNotExist/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('../test_fixtures/api-gateway-implementation.json', 'thisFolderDoesNotExist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -60,7 +62,7 @@ describe('validate', () => { }); it('exits with error when the pattern instantiation file cannot be found in the input path', async () => { - await expect(validate('../doesNotExists/api-gateway-implementation.json', 'test_fixtures/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('../doesNotExists/api-gateway-implementation.json', 'test_fixtures/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -68,7 +70,7 @@ describe('validate', () => { }); it('exits with error when the pattern instantiation file does not contain JSON', async () => { - await expect(validate('test_fixtures/api-gateway-implementation.json', 'test_fixtures/markdown.md', metaSchemaLocation, debugDisabled)) + await expect(validate('test_fixtures/api-gateway-implementation.json', 'test_fixtures/markdown.md', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -78,7 +80,7 @@ describe('validate', () => { it('exits with error when the JSON Schema pattern URL returns a 404', async () => { fetchMock.mock('http://does-not-exist/api-gateway.json', 404); - await expect(validate('https://does-not-exist/api-gateway-implementation.json', 'http://does-not-exist/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('https://does-not-exist/api-gateway-implementation.json', 'http://does-not-exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -92,7 +94,7 @@ describe('validate', () => { fetchMock.mock('http://exist/api-gateway.json', apiGateway); fetchMock.mock('https://does-not-exist/api-gateway-implementation.json', 404); - await expect(validate('https://does-not-exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('https://does-not-exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -107,7 +109,7 @@ describe('validate', () => { fetchMock.mock('http://exist/api-gateway.json', apiGateway); fetchMock.mock('https://url/with/non/json/response', markdown); - await expect(validate('https://url/with/non/json/response', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('https://url/with/non/json/response', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -124,7 +126,7 @@ describe('validate', () => { const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation-that-does-not-match-schema.json'), 'utf8'); fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); - await expect(validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -152,7 +154,7 @@ describe('validate', () => { const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation-that-does-not-pass-the-spectral-validation.json'), 'utf8'); fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); - await expect(validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -160,7 +162,7 @@ describe('validate', () => { fetchMock.restore(); }); - it('exits with error when the pattern does not pass all the spectral validations', async () => { + it('exits with error when the pattern does not pass all the spectral validations ', async () => { const apiGateway = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-with-no-relationships.json'), 'utf8'); fetchMock.mock('http://exist/api-gateway.json', apiGateway); @@ -168,7 +170,7 @@ describe('validate', () => { const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8'); fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); - await expect(validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled)) + await expect(validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); @@ -176,22 +178,35 @@ describe('validate', () => { }); it('exits with error when the meta schema location is not a directory', async () => { - await expect(validate('https://url/with/non/json/response', 'http://exist/api-gateway.json', 'test_fixtures/api-gateway.json', debugDisabled)) + await expect(validate('https://url/with/non/json/response', 'http://exist/api-gateway.json', 'test_fixtures/api-gateway.json', debugDisabled, jsonFormat)) .rejects .toThrow(); expect(mockExit).toHaveBeenCalledWith(1); }); - it('exits with error when the test report output location is not an xml file', async () => { - await expect(validate('test_fixtures/api-gateway-implementation.json', 'test_fixtures/api-gateway.json', metaSchemaLocation, debugDisabled, 'not-xml.json')) - .rejects - .toThrow(); + it('complete successfully when the pattern instantiation validates against the pattern json schema in Json format', async () => { + const mockExit = jest.spyOn(process, 'exit') + .mockImplementation((code) => { + if (code != 0) { + throw new Error(); + } + return undefined as never; + }); - expect(mockExit).toHaveBeenCalledWith(1); + const apiGateway = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway.json'), 'utf8'); + fetchMock.mock('http://exist/api-gateway.json', apiGateway); + + const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8'); + fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); + + await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat); + + expect(mockExit).toHaveBeenCalledWith(0); + fetchMock.restore(); }); - it('complete successfully when the pattern instantiation validates against the pattern json schema', async () => { + it('complete successfully when the pattern instantiation validates against the pattern json schema in JUnit format', async () => { const mockExit = jest.spyOn(process, 'exit') .mockImplementation((code) => { if (code != 0) { @@ -206,13 +221,13 @@ describe('validate', () => { const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8'); fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); - await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled); + await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jUnitFormat); expect(mockExit).toHaveBeenCalledWith(0); fetchMock.restore(); }); - it('complete successfully when the pattern instantiation validates against the pattern json schema and the test report is created', async () => { + it('complete successfully when the pattern instantiation validates against the pattern json schema and the JUnit format output file is created', async () => { const mockExit = jest.spyOn(process, 'exit') .mockImplementation((code) => { if (code != 0) { @@ -229,7 +244,7 @@ describe('validate', () => { const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'report')); const testReportLocation = path.join(tmpDir, 'test-report.xml'); - await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, testReportLocation); + await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jUnitFormat, testReportLocation); expect(existsSync(testReportLocation)).toBe(true); //check that the test report exists @@ -237,8 +252,33 @@ describe('validate', () => { expect(mockExit).toHaveBeenCalledWith(0); fetchMock.restore(); + }); + it('complete successfully when the pattern instantiation validates against the pattern json schema and the JSON format output file is created', async () => { + const mockExit = jest.spyOn(process, 'exit') + .mockImplementation((code) => { + if (code != 0) { + throw new Error(); + } + return undefined as never; + }); + + const apiGateway = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway.json'), 'utf8'); + fetchMock.mock('http://exist/api-gateway.json', apiGateway); + const apiGatewayInstantiation = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-implementation.json'), 'utf8'); + fetchMock.mock('https://exist/api-gateway-implementation.json', apiGatewayInstantiation); + + const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'report')); + const reportLocation = path.join(tmpDir, 'report.json'); + await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat, reportLocation); + + expect(existsSync(reportLocation)).toBe(true); //check that the test report exists + + rmSync(tmpDir, {recursive: true}); //delete folder with the test report + + expect(mockExit).toHaveBeenCalledWith(0); + fetchMock.restore(); }); }); diff --git a/cli/src/commands/validate/validate.ts b/cli/src/commands/validate/validate.ts index 1a71dfa3..e861b399 100644 --- a/cli/src/commands/validate/validate.ts +++ b/cli/src/commands/validate/validate.ts @@ -1,5 +1,5 @@ import Ajv2020, { ErrorObject } from 'ajv/dist/2020.js'; -import { existsSync, promises as fs, readFileSync, readdirSync, statSync } from 'fs'; +import { existsSync, promises as fs, readFileSync, readdirSync, statSync, writeFileSync } from 'fs'; import pkg, { ISpectralDiagnostic } from '@stoplight/spectral-core'; const { Spectral } = pkg; import { getRuleset } from '@stoplight/spectral-cli/dist/services/linter/utils/getRuleset.js'; @@ -10,10 +10,19 @@ import { ValidationOutput as ValidationOutput } from './validation.output.js'; import { SpectralResult } from './spectral.result.js'; import createJUnitReport from './junit-report/junit.report.js'; import yaml from 'js-yaml'; +import path from 'path'; +import { mkdirp } from 'mkdirp'; let logger: winston.Logger; // defined later at startup -export default async function validate(jsonSchemaInstantiationLocation: string, jsonSchemaLocation: string, metaSchemaPath: string, debug: boolean = false, junitReportLocation?: string) { +export default async function validate( + jsonSchemaInstantiationLocation: string, + jsonSchemaLocation: string, + metaSchemaPath: string, + debug: boolean = false, + format: string, + output?: string +) { logger = initLogger(debug); let errors = false; let validations: ValidationOutput[] = []; @@ -45,25 +54,20 @@ export default async function validate(jsonSchemaInstantiationLocation: string, validations = validations.concat(jsonSchemaValidations); } - if(junitReportLocation) { - if (!junitReportLocation.endsWith('.xml')){ - throw new Error('The test report location must be an xml file'); - } - logger.debug('Generating test report file'); - const spectralRules = extractRulesFromSpectralRulesets(spectralRulesetForInstantiation, spectralRulesetForPattern); - createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, spectralRules, junitReportLocation); - } + const validationsOutput = getFormattedOutput(validations, format, spectralRulesetForInstantiation, spectralRulesetForPattern, jsonSchemaValidations, spectralResult); + + createOutputFile(output, validationsOutput); if(errors){ - logger.error(`The following issues have been found on the JSON Schema instantiation ${prettifyJson(validations)}`); + logger.error(`The following issues have been found on the JSON Schema instantiation ${validationsOutput}`); process.exit(1); } - - if(validations.length > 0){ - logger.info(`The following issues (not errors) have been found on the JSON Schema Instantiation ${prettifyJson(validations)}`); - }else{ - logger.info('The JSON Schema instantiation is valid'); + + logger.info('The JSON Schema instantiation is valid'); + if(validationsOutput != '[]'){ + logger.info(validationsOutput); } + process.exit(0); } catch (error) { @@ -72,6 +76,31 @@ export default async function validate(jsonSchemaInstantiationLocation: string, } } +function getFormattedOutput( + validations: ValidationOutput[], + format: string, + spectralRulesetForInstantiation: string, + spectralRulesetForPattern: string, + jsonSchemaValidations: ValidationOutput[], + spectralResult: SpectralResult +) { + if (format === 'junit') { + const spectralRules = extractRulesFromSpectralRulesets(spectralRulesetForInstantiation, spectralRulesetForPattern); + return createJUnitReport(jsonSchemaValidations, spectralResult.spectralIssues, spectralRules); + } + return prettifyJson(validations); + +} + +function createOutputFile(output: string, validationsOutput: string) { + if (output) { + logger.debug('Creating report'); + const dirname = path.dirname(output); + mkdirp.sync(dirname); + writeFileSync(output, validationsOutput); + } +} + function buildAjv2020(debug: boolean): Ajv2020 { if (debug){ return new Ajv2020({strict: 'log', allErrors: true}); diff --git a/cli/src/index.ts b/cli/src/index.ts index 8673cf7f..744b0c87 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -42,10 +42,15 @@ program .requiredOption('-p, --pattern ', 'Path to the pattern file to use. May be a file path or a URL.') .requiredOption('-i, --instantiation ', 'Path to the pattern instantiation file to use. May be a file path or a URL.') .option('-m, --metaSchemasLocation ', 'The location of the directory of the meta schemas to be loaded', '../calm/draft/2024-03/meta') - .option('-tr, --test-report ', 'Path location at which to output the generated test report.') + .addOption( + new Option('-f, --format ', 'The format of the output') + .choices(['json', 'junit']) + .default('json') + ) + .option('-o, --output ', 'Path location at which to output the generated file.') .option('-v, --verbose', 'Enable verbose logging.', false) .action(async (options) => - await validate(options.instantiation, options.pattern, options.metaSchemasLocation, options.verbose, options.testReport) + await validate(options.instantiation, options.pattern, options.metaSchemasLocation, options.verbose, options.format, options.output) ); program.parse(process.argv); From cfe08c11d74e203a59ee398a89bf8005858ae479 Mon Sep 17 00:00:00 2001 From: Luigi Bulanti Date: Wed, 1 May 2024 11:44:21 +0100 Subject: [PATCH 15/15] issue-98 - refactoring test --- cli/src/commands/validate/validate.spec.ts | 30 +++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cli/src/commands/validate/validate.spec.ts b/cli/src/commands/validate/validate.spec.ts index cc17f0aa..dfca709d 100644 --- a/cli/src/commands/validate/validate.spec.ts +++ b/cli/src/commands/validate/validate.spec.ts @@ -45,10 +45,16 @@ describe('validate', () => { beforeEach(() => { mockRunFunction.mockReturnValue([]); mockExit = jest.spyOn(process, 'exit') - .mockImplementation((code) => { throw new Error(`The exit code is ${code}`); }); + .mockImplementation((code) => { + if (code != 0) { + throw new Error(); + } + return undefined as never; + }); }); afterEach(() => { + fetchMock.restore(); mockExit.mockRestore(); }); @@ -85,7 +91,6 @@ describe('validate', () => { .toThrow(); expect(mockExit).toHaveBeenCalledWith(1); - fetchMock.restore(); }); it('exits with error when the pattern instantiation URL returns a 404', async () => { @@ -99,7 +104,6 @@ describe('validate', () => { .toThrow(); expect(mockExit).toHaveBeenCalledWith(1); - fetchMock.restore(); }); it('exits with error when the pattern instantiation file at given URL returns non JSON response', async () => { @@ -114,7 +118,6 @@ describe('validate', () => { .toThrow(); expect(mockExit).toHaveBeenCalledWith(1); - fetchMock.restore(); }); @@ -131,7 +134,6 @@ describe('validate', () => { .toThrow(); expect(mockExit).toHaveBeenCalledWith(1); - fetchMock.restore(); }); it('exits with error when the pattern instantiation does not pass all the spectral validations', async () => { @@ -159,10 +161,20 @@ describe('validate', () => { .toThrow(); expect(mockExit).toHaveBeenCalledWith(1); - fetchMock.restore(); }); it('exits with error when the pattern does not pass all the spectral validations ', async () => { + const expectedSpectralOutput: ISpectralDiagnostic[] = [ + { + code: 'no-empty-properties', + message: 'Must not contain string properties set to the empty string or numerical properties set to zero', + severity: 0, + path: JSON.parse(JSON.stringify('$..*')), + range: { start: { line: 1, character: 1 }, end: { line: 2, character: 1 } } + } + ]; + + mockRunFunction.mockReturnValue(expectedSpectralOutput); const apiGateway = readFileSync(path.resolve(__dirname, '../../../test_fixtures/api-gateway-with-no-relationships.json'), 'utf8'); fetchMock.mock('http://exist/api-gateway.json', apiGateway); @@ -173,8 +185,6 @@ describe('validate', () => { await expect(validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat)) .rejects .toThrow(); - - fetchMock.restore(); }); it('exits with error when the meta schema location is not a directory', async () => { @@ -203,7 +213,6 @@ describe('validate', () => { await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jsonFormat); expect(mockExit).toHaveBeenCalledWith(0); - fetchMock.restore(); }); it('complete successfully when the pattern instantiation validates against the pattern json schema in JUnit format', async () => { @@ -224,7 +233,6 @@ describe('validate', () => { await validate('https://exist/api-gateway-implementation.json', 'http://exist/api-gateway.json', metaSchemaLocation, debugDisabled, jUnitFormat); expect(mockExit).toHaveBeenCalledWith(0); - fetchMock.restore(); }); it('complete successfully when the pattern instantiation validates against the pattern json schema and the JUnit format output file is created', async () => { @@ -251,7 +259,6 @@ describe('validate', () => { rmSync(tmpDir, {recursive: true}); //delete folder with the test report expect(mockExit).toHaveBeenCalledWith(0); - fetchMock.restore(); }); it('complete successfully when the pattern instantiation validates against the pattern json schema and the JSON format output file is created', async () => { @@ -278,7 +285,6 @@ describe('validate', () => { rmSync(tmpDir, {recursive: true}); //delete folder with the test report expect(mockExit).toHaveBeenCalledWith(0); - fetchMock.restore(); }); });