diff --git a/cli/README.md b/cli/README.md index 9bc69540..bb1e74fa 100644 --- a/cli/README.md +++ b/cli/README.md @@ -62,10 +62,11 @@ Usage: calm generate [options] Generate an instantiation from a CALM pattern file. Options: - -p, --pattern Path to the pattern file to use. May be a file path or a URL. - -o, --output Path location at which to output the generated file. - -v, --verbose Enable verbose logging. (default: false) - -h, --help display help for command + -p, --pattern Path to the pattern file to use. May be a file path or a URL. + -o, --output Path location at which to output the generated file. + -s, --schemaDirectory Path to a directory of schemas to be used when instantiating patterns. + -v, --verbose Enable verbose logging. (default: false) + -h, --help display help for command ``` ### Validating a CALM instantiation diff --git a/cli/package-lock.json b/cli/package-lock.json index f37bf2dd..4324af8c 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -15,6 +15,8 @@ "commander": "^12.0.0", "fetch-mock": "^9.11.0", "graphviz-cli": "^2.0.0", + "json-pointer": "^0.6.2", + "lodash": "^4.17.21", "mkdirp": "^3.0.1", "ts-graphviz": "^2.1.1", "tsconfig-paths": "^4.2.0", @@ -26,6 +28,8 @@ "devDependencies": { "@jest/globals": "^29.7.0", "@types/jest": "^29.5.12", + "@types/json-pointer": "^1.0.34", + "@types/lodash": "^4.17.0", "@types/node": "^20.11.30", "@typescript-eslint/eslint-plugin": "^6.21.0", "eslint": "^8.57.0", @@ -1962,6 +1966,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/json-pointer": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/@types/json-pointer/-/json-pointer-1.0.34.tgz", + "integrity": "sha512-JRnWcxzXSaLei98xgw1B7vAeBVOrkyw0+Rt9j1QoJrczE78OpHsyQC8GNbuhw+/2vxxDe58QvWnngS86CoIbRg==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.15", "license": "MIT" @@ -1971,6 +1981,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/lodash": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz", + "integrity": "sha512-t7dhREVv6dbNj0q17X12j7yDG4bD/DHYX7o5/DbDxobP0HnGPgpRz2Ej77aL7TZT3DSw13fqUTj8J4mMnqa7WA==", + "dev": true + }, "node_modules/@types/node": { "version": "20.11.30", "license": "MIT", @@ -3991,6 +4007,11 @@ "is-callable": "^1.1.3" } }, + "node_modules/foreach": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.6.tgz", + "integrity": "sha512-k6GAGDyqLe9JaebCsFCoudPPWfihKu8pylYXRlqP1J7ms39iPoTtk2fviNglIeQEwdh0bQeKJ01ZPyuyQvKzwg==" + }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", @@ -5353,6 +5374,14 @@ "dev": true, "license": "MIT" }, + "node_modules/json-pointer": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/json-pointer/-/json-pointer-0.6.2.tgz", + "integrity": "sha512-vLWcKbOaXlO+jvRy4qNd+TI1QUPZzfJj1tpJ3vAXDych5XJf93ftpUKe5pKCrzyIIwgBJcOcCVRUfqQP25afBw==", + "dependencies": { + "foreach": "^2.0.4" + } + }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -5457,7 +5486,8 @@ }, "node_modules/lodash": { "version": "4.17.21", - "license": "MIT" + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.isequal": { "version": "4.5.0", @@ -6752,8 +6782,9 @@ }, "node_modules/ts-jest": { "version": "29.1.2", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.2.tgz", + "integrity": "sha512-br6GJoH/WUX4pu7FbZXuWGKGNDuU7b8Uj77g/Sp7puZV6EXzuByl6JrECvm0MzVzSTkSHWTihsXt+5XYER5b+g==", "dev": true, - "license": "MIT", "dependencies": { "bs-logger": "0.x", "fast-json-stable-stringify": "2.x", diff --git a/cli/package.json b/cli/package.json index 69e31363..c968be39 100644 --- a/cli/package.json +++ b/cli/package.json @@ -25,6 +25,8 @@ "commander": "^12.0.0", "fetch-mock": "^9.11.0", "graphviz-cli": "^2.0.0", + "json-pointer": "^0.6.2", + "lodash": "^4.17.21", "mkdirp": "^3.0.1", "ts-graphviz": "^2.1.1", "tsconfig-paths": "^4.2.0", @@ -33,6 +35,8 @@ "devDependencies": { "@jest/globals": "^29.7.0", "@types/jest": "^29.5.12", + "@types/json-pointer": "^1.0.34", + "@types/lodash": "^4.17.0", "@types/node": "^20.11.30", "@typescript-eslint/eslint-plugin": "^6.21.0", "eslint": "^8.57.0", diff --git a/cli/src/commands/generate/components/node.spec.ts b/cli/src/commands/generate/components/node.spec.ts new file mode 100644 index 00000000..00086beb --- /dev/null +++ b/cli/src/commands/generate/components/node.spec.ts @@ -0,0 +1,259 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { SchemaDirectory } from '../schema-directory'; +import { instantiateNodeInterfaces, instantiateNodes } from './node'; + +jest.mock('../../helper', () => { + return { + initLogger: () => { + return { + info: () => { }, + debug: () => { } + }; + } + }; +}); + +jest.mock('../schema-directory'); + +let mockSchemaDir; + +beforeEach(() => { + mockSchemaDir = new SchemaDirectory('directory'); +}); + +function getSamplePatternNode(properties: any): any { + return { + properties: { + nodes: { + type: 'array', + prefixItems: [ + { + properties: properties + } + ] + } + } + }; +} + + +describe('instantiateNodes', () => { + it('return instantiated node with array property', () => { + const pattern = getSamplePatternNode({ + 'property-name': { + type: 'array' + } + }); + expect(instantiateNodes(pattern, mockSchemaDir)) + .toEqual( + [{ + 'property-name': [ + '{{ PROPERTY_NAME }}' + ] + }] + ); + }); + + it('return instantiated node with string property', () => { + const pattern = getSamplePatternNode({ + 'property-name': { + type: 'string' + } + }); + + expect(instantiateNodes(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': '{{ PROPERTY_NAME }}' + } + ]); + }); + + it('return instantiated node with const property', () => { + const pattern = getSamplePatternNode({ + 'property-name': { + const: 'value here' + } + }); + + expect(instantiateNodes(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': 'value here' + } + ]); + }); + + it('call schema directory to resolve $ref nodes`', () => { + const reference = 'https://calm.com/core.json#/node'; + const pattern = { + properties: { + nodes: { + type: 'array', + prefixItems: [ + { + '$ref': reference + } + ] + } + } + }; + + const spy = jest.spyOn(mockSchemaDir, 'getDefinition'); + spy.mockReturnValue({ + properties: { + 'property-name': { + const: 'value here' + } + } + }); + + expect(instantiateNodes(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': 'value here' + } + ]); + expect(spy).toHaveBeenCalledWith(reference); + }); + + it('return instantiated node with interface', () => { + const pattern = { + properties: { + nodes: { + type: 'array', + prefixItems: [ + { + properties: { + 'unique-id': { + 'const': 'unique-id' + }, + // interfaces should be inserted + 'interfaces': { + 'prefixItems': [ + { + properties: { + // should insert placeholder {{ INTERFACE_PROPERTY }} + 'interface-property': { + 'type': 'string' + } + } + } + ] + } + } + } + ] + } + + } + + }; + + const expected = [ + { + 'unique-id': 'unique-id', + 'interfaces': [ + { + 'interface-property': '{{ INTERFACE_PROPERTY }}' + } + ] + } + ]; + + expect(instantiateNodes(pattern, mockSchemaDir)) + .toEqual(expected); + + }); +}); + + +function getSampleNodeInterfaces(properties: any): any { + return { + prefixItems: [ + { + properties: properties + } + ] + }; +} + + +describe('instantiateNodeInterfaces', () => { + + it('return instantiated node with array property', () => { + const pattern = getSampleNodeInterfaces({ + 'property-name': { + type: 'array' + } + }); + expect(instantiateNodeInterfaces(pattern, mockSchemaDir)) + .toEqual( + [{ + 'property-name': [ + '{{ PROPERTY_NAME }}' + ] + }] + ); + }); + + it('return instantiated node with string property', () => { + const pattern = getSampleNodeInterfaces({ + 'property-name': { + type: 'string' + } + }); + + expect(instantiateNodeInterfaces(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': '{{ PROPERTY_NAME }}' + } + ]); + }); + + it('return instantiated node with const property', () => { + const pattern = getSampleNodeInterfaces({ + 'property-name': { + const: 'value here' + } + }); + + expect(instantiateNodeInterfaces(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': 'value here' + } + ]); + }); + + it('call schema directory to resolve $ref interfaces`', () => { + const reference = 'https://calm.com/core.json#/interface'; + + const pattern = { + prefixItems: [ + { + '$ref': reference + } + ] + }; + + const spy = jest.spyOn(mockSchemaDir, 'getDefinition'); + spy.mockReturnValue({ + properties: { + 'property-name': { + const: 'value here' + } + } + }); + + expect(instantiateNodeInterfaces(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': 'value here' + } + ]); + expect(spy).toHaveBeenCalledWith(reference); + }); +}); \ No newline at end of file diff --git a/cli/src/commands/generate/components/node.ts b/cli/src/commands/generate/components/node.ts new file mode 100644 index 00000000..601d0184 --- /dev/null +++ b/cli/src/commands/generate/components/node.ts @@ -0,0 +1,113 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { initLogger } from '../../helper.js'; +import { SchemaDirectory } from '../schema-directory.js'; +import { mergeSchemas } from '../util.js'; +import { getPropertyValue } from './property.js'; + +/** + * Instantiate an individual node from its definition, resolving $refs if appropriate. + * @param nodeDef The definition of the node. + * @param schemaDirectory The schema directory to resolve refs against. + * @param debug Whether to log debug detail. + * @returns An instantiated node. + */ +export function instantiateNode(nodeDef: any, schemaDirectory: SchemaDirectory, debug: boolean = false): any { + const logger = initLogger(debug); + let fullDefinition = nodeDef; + if (nodeDef['$ref']) { + const ref = nodeDef['$ref']; + const schemaDef = schemaDirectory.getDefinition(ref); + + fullDefinition = mergeSchemas(schemaDef, nodeDef); + } + logger.debug('Generating node from ' + JSON.stringify(fullDefinition)); + + if (!('properties' in fullDefinition)) { + return {}; + } + + const out = {}; + for (const [key, detail] of Object.entries(fullDefinition['properties'])) { + if (key === 'interfaces') { + const interfaces = instantiateNodeInterfaces(detail, schemaDirectory, debug); + out['interfaces'] = interfaces; + } + else { + out[key] = getPropertyValue(key, detail); + } + } + return out; +} + +/** + * Instantiate all nodes in the document. + * @param pattern The pattern object to instantiate nodes from. + * @param schemaDirectory The schema directory to resolve refs against. + * @param debug Whether to log debug detail. + * @returns An array of instantiated nodes. + */ +export function instantiateNodes(pattern: any, schemaDirectory: SchemaDirectory, debug: boolean = false): any { + const logger = initLogger(debug); + const nodes = pattern?.properties?.nodes?.prefixItems; + if (!nodes) { + logger.error('Warning: pattern has no nodes defined.'); + if (pattern?.properties?.nodes?.items) { + logger.warn('Note: properties.relationships.items is deprecated: please use prefixItems instead.'); + } + return []; + } + const outputNodes = []; + + for (const node of nodes) { + outputNodes.push(instantiateNode(node, schemaDirectory)); + } + return outputNodes; +} + +/** + * Instantiate an individual interface on a node. + * @param interfaceDef The definition of the interface. + * @param schemaDirectory The schema directory to resolve refs against. + * @param debug Whether to log debug detail. + * @returns An instantiated interface. + */ +export function instantiateInterface(interfaceDef: object, schemaDirectory: SchemaDirectory, debug: boolean = false): object { + const logger = initLogger(debug); + let fullDefinition = interfaceDef; + if (interfaceDef['$ref']) { + const ref = interfaceDef['$ref']; + const schemaDef = schemaDirectory.getDefinition(ref); + + fullDefinition = mergeSchemas(schemaDef, interfaceDef); + } + + logger.debug('Generating interface from ' + JSON.stringify(fullDefinition, undefined, 2)); + + if (!('properties' in fullDefinition)) { + return {}; + } + + const out = {}; + for (const [key, detail] of Object.entries(fullDefinition['properties'])) { + out[key] = getPropertyValue(key, detail); + } + + return out; +} + +export function instantiateNodeInterfaces(detail: any, schemaDirectory: SchemaDirectory, debug: boolean = false): any[] { + const logger = initLogger(debug); + const interfaces = []; + if (!('prefixItems' in detail)) { + logger.error('No items in interfaces block.'); + return []; + } + + const interfaceDefs = detail.prefixItems; + for (const interfaceDef of interfaceDefs) { + interfaces.push(instantiateInterface(interfaceDef, schemaDirectory, debug)); + } + + return interfaces; +} \ No newline at end of file diff --git a/cli/src/commands/generate/components/property.spec.ts b/cli/src/commands/generate/components/property.spec.ts new file mode 100644 index 00000000..4963c8ed --- /dev/null +++ b/cli/src/commands/generate/components/property.spec.ts @@ -0,0 +1,63 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { getPropertyValue } from './property'; + +jest.mock('../../helper', () => { + return { + initLogger: () => { + return { + info: () => {}, + debug: () => {} + }; + } + }; +}); + +describe('getPropertyValue', () => { + it('generates string placeholder name from variable', () => { + expect(getPropertyValue('key-name', { + 'type': 'string' + })) + .toBe('{{ KEY_NAME }}'); + }); + + it('generates integer placeholder from variable', () => { + expect(getPropertyValue('key-name', { + 'type': 'integer' + })) + .toBe(-1); + }); + + it('generates const value if const is provided', () => { + expect(getPropertyValue('key-name', { + 'const': 'Example value' + })) + .toBe('Example value'); + }); + + it('generates const value with entire subtree if const is provided', () => { + expect(getPropertyValue('key-name', { + 'const': { + 'connects': { + 'source': 'source', + 'destination': 'destination' + } + } + })) + .toEqual({ + 'connects': { + 'source': 'source', + 'destination': 'destination' + } + }); + }); + + it('generates array with single string placeholder', () => { + expect(getPropertyValue('key-name', { + 'type': 'array' + })) + .toEqual([ + '{{ KEY_NAME }}' + ]); + }); +}); \ No newline at end of file diff --git a/cli/src/commands/generate/components/property.ts b/cli/src/commands/generate/components/property.ts new file mode 100644 index 00000000..ac659372 --- /dev/null +++ b/cli/src/commands/generate/components/property.ts @@ -0,0 +1,34 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +export function getStringPlaceholder(name: string): string { + return '{{ ' + name.toUpperCase().replaceAll('-', '_') + ' }}'; +} + +export function getPropertyValue(keyName: string, detail: any): any { + // TODO follow refs here + // should be able to instantiate not just a simple enum type but also a whole sub-object + // if both const and type are defined, prefer const + if ('const' in detail) { + return detail['const']; + } + + if ('type' in detail) { + const propertyType = detail['type']; + + if (propertyType === 'string') { + return getStringPlaceholder(keyName); + } + if (propertyType === 'integer') { + return -1; + } + if (propertyType === 'array') { + return [ + getStringPlaceholder(keyName) + ]; + } + } + + if ('$ref' in detail) { + console.log('Not following $ref on property, implementation TODO'); + } +} \ No newline at end of file diff --git a/cli/src/commands/generate/components/relationship.spec.ts b/cli/src/commands/generate/components/relationship.spec.ts new file mode 100644 index 00000000..55b8c6fe --- /dev/null +++ b/cli/src/commands/generate/components/relationship.spec.ts @@ -0,0 +1,121 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { SchemaDirectory } from '../schema-directory'; +import { instantiateRelationships } from './relationship'; + +jest.mock('../../helper', () => { + return { + initLogger: () => { + return { + info: () => {}, + debug: () => {} + }; + } + }; +}); + +jest.mock('../schema-directory'); + +let mockSchemaDir; + +beforeEach(() => { + mockSchemaDir = new SchemaDirectory('directory'); +}); + +function getSamplePatternRelationship(properties: any): any { + return { + properties: { + relationships: { + type: 'array', + prefixItems: [ + { + properties: properties + } + ] + } + } + }; +} + +describe('instantiateRelationships', () => { + + it('return instantiated relationship with array property', () => { + const pattern = getSamplePatternRelationship({ + 'property-name': { + type: 'array' + } + }); + + expect(instantiateRelationships(pattern, mockSchemaDir)) + .toEqual( + [{ + 'property-name': [ + '{{ PROPERTY_NAME }}' + ] + }] + ); + }); + + it('return instantiated relationship with string property', () => { + const pattern = getSamplePatternRelationship({ + 'property-name': { + type: 'string' + } + }); + + expect(instantiateRelationships(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': '{{ PROPERTY_NAME }}' + } + ]); + }); + + it('return instantiated relationship with const property', () => { + const pattern = getSamplePatternRelationship({ + 'property-name': { + const: 'value here' + } + }); + + expect(instantiateRelationships(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': 'value here' + } + ]); + }); + + it('call schema directory to resolve $ref relationships`', () => { + const reference = 'https://calm.com/core.json#/relationship'; + const pattern = { + properties: { + relationships: { + type: 'array', + prefixItems: [ + { + '$ref': reference + } + ] + } + } + }; + + const spy = jest.spyOn(mockSchemaDir, 'getDefinition'); + spy.mockReturnValue({ + properties: { + 'property-name': { + const: 'value here' + } + } + }); + + expect(instantiateRelationships(pattern, mockSchemaDir)) + .toEqual([ + { + 'property-name': 'value here' + } + ]); + expect(spy).toHaveBeenCalledWith(reference); + }); +}); \ No newline at end of file diff --git a/cli/src/commands/generate/components/relationship.ts b/cli/src/commands/generate/components/relationship.ts new file mode 100644 index 00000000..27c59823 --- /dev/null +++ b/cli/src/commands/generate/components/relationship.ts @@ -0,0 +1,65 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { initLogger } from '../../helper.js'; +import { SchemaDirectory } from '../schema-directory.js'; +import { mergeSchemas } from '../util.js'; +import { getPropertyValue } from './property.js'; + + +/** + * Instantiates an individual relationship. + * @param relationshipDef The relationship definition to instantiate + * @param schemaDirectory The schema directory to resolve refs against. + * @param debug Whether to log debug detail + * @returns An instantiated relationship. + */ +function instantiateRelationship(relationshipDef: object, schemaDirectory: SchemaDirectory, debug: boolean = false): object { + const logger = initLogger(debug); + let fullDefinition = relationshipDef; + if (relationshipDef['$ref']) { + const ref = relationshipDef['$ref']; + const schemaDef = schemaDirectory.getDefinition(ref); + + fullDefinition = mergeSchemas(schemaDef, relationshipDef); + } + + if (!('properties' in fullDefinition)) { + return {}; + } + + logger.debug('Generating interface from ' + JSON.stringify(fullDefinition, undefined, 2)); + + const out = {}; + for (const [key, detail] of Object.entries(fullDefinition['properties'])) { + out[key] = getPropertyValue(key, detail); + } + + return out; +} + +/** + * Instantiate all relationships in the document. + * @param pattern The pattern object to instantiate relationships from. + * @param schemaDirectory The schema directory to resolve refs against. + * @param debug Whether to log debug detail. + * @returns An array of instantiated relationships. + */ +export function instantiateRelationships(pattern: any, schemaDirectory: SchemaDirectory, debug: boolean = false): any { + const logger = initLogger(debug); + const relationships = pattern?.properties?.relationships?.prefixItems; + + if (!relationships) { + logger.error('Warning: pattern has no relationships defined'); + if (pattern?.properties?.relationships?.items) { + logger.warn('Note: properties.relationships.items is deprecated: please use prefixItems instead.'); + } + return []; + } + + const outputRelationships = []; + for (const relationship of relationships) { + outputRelationships.push(instantiateRelationship(relationship, schemaDirectory, debug)); + } + + return outputRelationships; +} \ No newline at end of file diff --git a/cli/src/commands/generate/generate.spec.ts b/cli/src/commands/generate/generate.spec.ts index 544c6a45..87767192 100644 --- a/cli/src/commands/generate/generate.spec.ts +++ b/cli/src/commands/generate/generate.spec.ts @@ -5,6 +5,7 @@ import { runGenerate } from './generate'; import { tmpdir } from 'node:os'; import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import path from 'node:path'; +import { SchemaDirectory } from './schema-directory'; jest.mock('../helper', () => { return { @@ -17,305 +18,19 @@ jest.mock('../helper', () => { }; }); -const { - getPropertyValue, - instantiateNodes, - instantiateRelationships, - instantiateNodeInterfaces, - instantiateAdditionalTopLevelProperties -} = exportedForTesting; +jest.mock('./schema-directory'); -describe('getPropertyValue', () => { - it('generates string placeholder name from variable', () => { - expect(getPropertyValue('key-name', { - 'type': 'string' - })) - .toBe('{{ KEY_NAME }}'); - }); +let mockSchemaDir; - it('generates integer placeholder from variable', () => { - expect(getPropertyValue('key-name', { - 'type': 'integer' - })) - .toBe(-1); - }); - - it('generates const value if const is provided', () => { - expect(getPropertyValue('key-name', { - 'const': 'Example value' - })) - .toBe('Example value'); - }); - - it('generates const value with entire subtree if const is provided', () => { - expect(getPropertyValue('key-name', { - 'const': { - 'connects': { - 'source': 'source', - 'destination': 'destination' - } - } - })) - .toEqual({ - 'connects': { - 'source': 'source', - 'destination': 'destination' - } - }); - }); - - it('generates array with single string placeholder', () => { - expect(getPropertyValue('key-name', { - 'type': 'array' - })) - .toEqual([ - '{{ KEY_NAME }}' - ]); - }); +beforeEach(() => { + mockSchemaDir = new SchemaDirectory('directory'); }); +const { + instantiateAdditionalTopLevelProperties +} = exportedForTesting; -function getSamplePatternNode(properties: any): any { - return { - properties: { - nodes: { - type: 'array', - prefixItems: [ - { - properties: properties - } - ] - } - } - }; -} - - -describe('instantiateNodes', () => { - it('return instantiated node with array property', () => { - const pattern = getSamplePatternNode({ - 'property-name': { - type: 'array' - } - }); - expect(instantiateNodes(pattern)) - .toEqual( - [{ - 'property-name': [ - '{{ PROPERTY_NAME }}' - ] - }] - ); - }); - - it('return instantiated node with string property', () => { - const pattern = getSamplePatternNode({ - 'property-name': { - type: 'string' - } - }); - - expect(instantiateNodes(pattern)) - .toEqual([ - { - 'property-name': '{{ PROPERTY_NAME }}' - } - ]); - }); - - it('return instantiated node with const property', () => { - const pattern = getSamplePatternNode({ - 'property-name': { - const: 'value here' - } - }); - - expect(instantiateNodes(pattern)) - .toEqual([ - { - 'property-name': 'value here' - } - ]); - }); - - it('return instantiated node with interface', () => { - const pattern = { - properties: { - nodes: { - type: 'array', - prefixItems: [ - { - properties: { - 'unique-id': { - 'const': 'unique-id' - }, - // interfaces should be inserted - 'interfaces': { - 'prefixItems': [ - { - properties: { - // should insert placeholder {{ INTERFACE_PROPERTY }} - 'interface-property': { - 'type': 'string' - } - } - } - ] - } - } - } - ] - } - - } - - }; - - const expected = [ - { - 'unique-id': 'unique-id', - 'interfaces': [ - { - 'interface-property': '{{ INTERFACE_PROPERTY }}' - } - ] - } - ]; - - expect(instantiateNodes(pattern)) - .toEqual(expected); - - }); -}); - - -function getSampleNodeInterfaces(properties: any): any { - return { - prefixItems: [ - { - properties: properties - } - ] - }; -} - - -describe('instantiateNodeInterfaces', () => { - - it('return instantiated node with array property', () => { - const pattern = getSampleNodeInterfaces({ - 'property-name': { - type: 'array' - } - }); - expect(instantiateNodeInterfaces(pattern)) - .toEqual( - [{ - 'property-name': [ - '{{ PROPERTY_NAME }}' - ] - }] - ); - }); - - it('return instantiated node with string property', () => { - const pattern = getSampleNodeInterfaces({ - 'property-name': { - type: 'string' - } - }); - - expect(instantiateNodeInterfaces(pattern)) - .toEqual([ - { - 'property-name': '{{ PROPERTY_NAME }}' - } - ]); - }); - - it('return instantiated node with const property', () => { - const pattern = getSampleNodeInterfaces({ - 'property-name': { - const: 'value here' - } - }); - - expect(instantiateNodeInterfaces(pattern)) - .toEqual([ - { - 'property-name': 'value here' - } - ]); - }); -}); - - -function getSamplePatternRelationship(properties: any): any { - return { - properties: { - relationships: { - type: 'array', - prefixItems: [ - { - properties: properties - } - ] - } - } - }; -} - -describe('instantiateRelationships', () => { - - it('return instantiated relationship with array property', () => { - const pattern = getSamplePatternRelationship({ - 'property-name': { - type: 'array' - } - }); - - expect(instantiateRelationships(pattern)) - .toEqual( - [{ - 'property-name': [ - '{{ PROPERTY_NAME }}' - ] - }] - ); - }); - - it('return instantiated relationship with string property', () => { - const pattern = getSamplePatternRelationship({ - 'property-name': { - type: 'string' - } - }); - - expect(instantiateRelationships(pattern)) - .toEqual([ - { - 'property-name': '{{ PROPERTY_NAME }}' - } - ]); - }); - - it('return instantiated relationship with const property', () => { - const pattern = getSamplePatternRelationship({ - 'property-name': { - const: 'value here' - } - }); - - expect(instantiateRelationships(pattern)) - .toEqual([ - { - 'property-name': 'value here' - } - ]); - }); -}); - describe('instantiateAdditionalTopLevelProperties', () => { it('instantiate an additional top level array property', () => { const pattern = { @@ -330,7 +45,7 @@ describe('instantiateAdditionalTopLevelProperties', () => { } }; - expect(instantiateAdditionalTopLevelProperties(pattern)) + expect(instantiateAdditionalTopLevelProperties(pattern, mockSchemaDir)) .toEqual({ 'extra-property': { values: [ '{{ VALUES }}' ] @@ -351,7 +66,7 @@ describe('instantiateAdditionalTopLevelProperties', () => { } }; - expect(instantiateAdditionalTopLevelProperties(pattern)) + expect(instantiateAdditionalTopLevelProperties(pattern, mockSchemaDir)) .toEqual({ 'extra': { 'extra-property': 'value here' @@ -372,7 +87,7 @@ describe('instantiateAdditionalTopLevelProperties', () => { } }; - expect(instantiateAdditionalTopLevelProperties(pattern)) + expect(instantiateAdditionalTopLevelProperties(pattern, mockSchemaDir)) .toEqual({ extra: { 'extra-property': '{{ EXTRA_PROPERTY }}' @@ -396,7 +111,7 @@ describe('runGenerate', () => { it('instantiates to given directory', () => { const outPath = path.join(tempDirectoryPath, 'output.json'); - runGenerate(testPath, outPath, false); + runGenerate(testPath, outPath, undefined, false); expect(existsSync(outPath)) .toBeTruthy(); @@ -404,7 +119,7 @@ describe('runGenerate', () => { it('instantiates to given directory with nested folders', () => { const outPath = path.join(tempDirectoryPath, 'output/test/output.json'); - runGenerate(testPath, outPath, false); + runGenerate(testPath, outPath, undefined, false); expect(existsSync(outPath)) .toBeTruthy(); @@ -412,7 +127,7 @@ describe('runGenerate', () => { it('instantiates to calm instantiation file', () => { const outPath = path.join(tempDirectoryPath, 'output.json'); - runGenerate(testPath, outPath, false); + runGenerate(testPath, outPath, undefined, false); expect(existsSync(outPath)) .toBeTruthy(); diff --git a/cli/src/commands/generate/generate.ts b/cli/src/commands/generate/generate.ts index 52a1fc76..30008d50 100644 --- a/cli/src/commands/generate/generate.ts +++ b/cli/src/commands/generate/generate.ts @@ -7,6 +7,9 @@ import { mkdirp } from 'mkdirp'; import * as winston from 'winston'; import { initLogger } from '../helper.js'; import { CALMInstantiation } from '../../types.js'; +import { SchemaDirectory } from './schema-directory.js'; +import { instantiateNode, instantiateNodes } from './components/node.js'; +import { instantiateRelationships } from './components/relationship.js'; let logger: winston.Logger; // defined later at startup @@ -22,120 +25,7 @@ function loadFile(path: string): any { } -function getStringPlaceholder(name: string): string { - return '{{ ' + name.toUpperCase().replaceAll('-', '_') + ' }}'; -} - -function getPropertyValue(keyName: string, detail: any): any { - if ('const' in detail) { - return detail['const']; - } - - if ('type' in detail) { - const propertyType = detail['type']; - - if (propertyType === 'string') { - return getStringPlaceholder(keyName); - } - if (propertyType === 'integer') { - return -1; - } - if (propertyType === 'array') { - return [ - getStringPlaceholder(keyName) - ]; - } - } -} - -function instantiateNodeInterfaces(detail: any): any[] { - const interfaces = []; - if (!('prefixItems' in detail)) { - logger.error('No items in interfaces block.'); - return []; - } - - const interfaceDefs = detail.prefixItems; - for (const interfaceDef of interfaceDefs) { - if (!('properties' in interfaceDef)) { - continue; - } - - const out = {}; - for (const [key, detail] of Object.entries(interfaceDef['properties'])) { - out[key] = getPropertyValue(key, detail); - } - - interfaces.push(out); - } - - return interfaces; -} - -function instantiateNode(node: any): any { - const out = {}; - for (const [key, detail] of Object.entries(node['properties'])) { - if (key === 'interfaces') { - const interfaces = instantiateNodeInterfaces(detail); - out['interfaces'] = interfaces; - } - else { - out[key] = getPropertyValue(key, detail); - } - } - return out; -} - -function instantiateNodes(pattern: any): any { - const nodes = pattern?.properties?.nodes?.prefixItems; - if (!nodes) { - logger.error('Warning: pattern has no nodes defined.'); - if (pattern?.properties?.nodes?.items) { - logger.warn('Note: properties.relationships.items is deprecated: please use prefixItems instead.'); - } - return []; - } - const outputNodes = []; - - for (const node of nodes) { - if (!('properties' in node)) { - continue; - } - - outputNodes.push(instantiateNode(node)); - } - return outputNodes; -} - -function instantiateRelationships(pattern: any): any { - const relationships = pattern?.properties?.relationships?.prefixItems; - - if (!relationships) { - logger.error('Warning: pattern has no relationships defined'); - if (pattern?.properties?.relationships?.items) { - logger.warn('Note: properties.relationships.items is deprecated: please use prefixItems instead.'); - } - return []; - } - - const outputRelationships = []; - for (const relationship of relationships) { - if (!('properties' in relationship)) { - continue; - } - - const out = {}; - for (const [key, detail] of Object.entries(relationship['properties'])) { - out[key] = getPropertyValue(key, detail); - } - - outputRelationships.push(out); - } - - return outputRelationships; -} - -function instantiateAdditionalTopLevelProperties(pattern: any): any { +function instantiateAdditionalTopLevelProperties(pattern: any, schemaDirectory: SchemaDirectory): any { const properties = pattern?.properties; if (!properties) { logger.error('Warning: pattern has no properties defined.'); @@ -149,27 +39,23 @@ function instantiateAdditionalTopLevelProperties(pattern: any): any { continue; } - // TODO - extraProperties[additionalProperty] = instantiateNode(detail); + // TODO handle generic top level properties, not just nodes + extraProperties[additionalProperty] = instantiateNode(detail, schemaDirectory); } return extraProperties; } export const exportedForTesting = { - getPropertyValue, - instantiateNodes, - instantiateRelationships, - instantiateNodeInterfaces, instantiateAdditionalTopLevelProperties }; -export function generate(patternPath: string, debug: boolean): CALMInstantiation { +export function generate(patternPath: string, schemaDirectory: SchemaDirectory, debug: boolean): CALMInstantiation { logger = initLogger(debug); const pattern = loadFile(patternPath); - const outputNodes = instantiateNodes(pattern); - const relationshipNodes = instantiateRelationships(pattern); - const additionalProperties = instantiateAdditionalTopLevelProperties(pattern); + const outputNodes = instantiateNodes(pattern, schemaDirectory, debug); + const relationshipNodes = instantiateRelationships(pattern, schemaDirectory, debug); + const additionalProperties = instantiateAdditionalTopLevelProperties(pattern, schemaDirectory); const final = { 'nodes': outputNodes, @@ -180,8 +66,14 @@ export function generate(patternPath: string, debug: boolean): CALMInstantiation return final; } -export function runGenerate(patternPath: string, outputPath: string, debug: boolean): void { - const final = generate(patternPath, debug); +export async function runGenerate(patternPath: string, outputPath: string, schemaDirectoryPath: string, debug: boolean): Promise { + const schemaDirectory = new SchemaDirectory(schemaDirectoryPath); + + if (schemaDirectoryPath) { + await schemaDirectory.loadSchemas(); + } + + const final = generate(patternPath, schemaDirectory, debug); const output = JSON.stringify(final, null, 2); logger.debug('Generated instantiation: ' + output); diff --git a/cli/src/commands/generate/schema-directory.spec.ts b/cli/src/commands/generate/schema-directory.spec.ts new file mode 100644 index 00000000..024cd489 --- /dev/null +++ b/cli/src/commands/generate/schema-directory.spec.ts @@ -0,0 +1,70 @@ +import { SchemaDirectory } from './schema-directory'; + +jest.mock('../helper', () => { + return { + initLogger: () => { + return { + info: () => {}, + debug: () => {}, + warn: () => {}, + error: () => {} + }; + } + }; +}); + +describe('SchemaDirectory', () => { + it('loads all specs from given directory including subdirectories', async () => { + const schemaDir = new SchemaDirectory('../calm/draft/2024-03'); + + await schemaDir.loadSchemas(); + expect(schemaDir.getLoadedSchemas().length).toBe(2); + }); + + it('resolves a reference from a loaded schema', async () => { + const schemaDir = new SchemaDirectory('../calm/draft/2024-03'); + + await schemaDir.loadSchemas(); + const nodeRef = 'https://raw.githubusercontent.com/finos-labs/architecture-as-code/main/calm/draft/2024-03/meta/core.json#/defs/node'; + const nodeDef = schemaDir.getDefinition(nodeRef); + + // node should have a required property of node-type + expect(nodeDef.required).toContain('node-type'); + }); + + it('recursively resolve references from a loaded schema', async () => { + const schemaDir = new SchemaDirectory('../calm/draft/2024-04'); + + await schemaDir.loadSchemas(); + const interfaceRef = 'https://raw.githubusercontent.com/finos-labs/architecture-as-code/main/calm/draft/2024-04/meta/interface.json#/defs/host-port-interface'; + const interfaceDef = schemaDir.getDefinition(interfaceRef); + + // this should include host and port, but also recursively include unique-id + expect(interfaceDef.properties).toHaveProperty('host'); + expect(interfaceDef.properties).toHaveProperty('port'); + expect(interfaceDef.properties).toHaveProperty('unique-id'); + }); + + it('resolve to warning message if schema is missing', async () => { + const schemaDir = new SchemaDirectory('../calm/draft/2024-04'); + + const interfaceRef = 'https://raw.githubusercontent.com/finos-labs/architecture-as-code/main/calm/draft/2024-04/meta/interface.json#/defs/host-port-interface'; + const interfaceDef = schemaDir.getDefinition(interfaceRef); + + // this should include host and port, but also recursively include unique-id + expect(interfaceDef.properties).toHaveProperty('missing-value'); + expect(interfaceDef.properties['missing-value']).toEqual('MISSING OBJECT, ref: ' + interfaceRef + ' could not be resolved'); + }); + + it('terminate early in the case of a circular reference', async () => { + const schemaDir = new SchemaDirectory('test_fixtures/recursive_refs'); + + await schemaDir.loadSchemas(); + const interfaceRef = 'https://calm.com/recursive.json#/$defs/top-level'; + const interfaceDef = schemaDir.getDefinition(interfaceRef); + + // this should include top-level and port. If circular refs are not handled properly this will crash the whole test by stack overflow + expect(interfaceDef.properties).toHaveProperty('top-level'); + expect(interfaceDef.properties).toHaveProperty('prop'); + }); +}); \ No newline at end of file diff --git a/cli/src/commands/generate/schema-directory.ts b/cli/src/commands/generate/schema-directory.ts new file mode 100644 index 00000000..c77c065a --- /dev/null +++ b/cli/src/commands/generate/schema-directory.ts @@ -0,0 +1,142 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { readdir, readFile } from 'fs/promises'; +import { join } from 'path'; +import pointer from 'json-pointer'; +import { mergeSchemas } from './util.js'; +import { Logger } from 'winston'; +import { initLogger } from '../helper.js'; + +/** + * Stores a directory of schemas and resolves references against that directory. + * Can merge objects recursively and will handle circular references. + */ +export class SchemaDirectory { + private readonly schemas: Map = new Map(); + private readonly logger: Logger; + + /** + * Initialise the SchemaDirectory. Does not load the schemas until loadSchemas is called. + * @param directoryPath The directory path from which to load schemas. All JSON and YAML files under this path will be loaded, including subfolders. + * @param debug Whether to log at debug level. + */ + constructor(private directoryPath: string, debug: boolean = false) { + this.logger = initLogger(debug); + } + + /** + * Load the schemas from the configured directory path. + */ + public async loadSchemas() { + this.logger.debug('Loading schemas from ' + this.directoryPath); + const files = await readdir(this.directoryPath, { recursive: true }); + + const schemaPaths = files.filter(str => str.match(/^.*(json|yaml|yml)$/)) + .map(schemaPath => join(this.directoryPath, schemaPath)); + + for (const schemaPath of schemaPaths) { + await this.loadSchema(schemaPath); + } + + this.logger.info(`Loaded ${this.schemas.size} schemas.`); + } + + private lookupDefinition(schemaId: string, ref: string) { + const schema = this.getSchema(schemaId); + if (!schema) { + return undefined; + } + return pointer.get(schema, ref); + } + + private getDefinitionRecursive(definitionReference: string, currentSchemaId: string, visitedDefinitions: string[]) { + const splitReference = definitionReference.split('#'); + let newSchemaId = splitReference[0]; + const ref = splitReference[1]; + visitedDefinitions.push(definitionReference); + + if (!newSchemaId) { + newSchemaId = currentSchemaId; + this.logger.debug(`Resolving reference ${ref} against current schema.`); + } + this.logger.debug(`Recursively resolving the reference, ref: ${ref}`); + const definition = this.lookupDefinition(newSchemaId, ref); + if (!definition) { + // schema not defined + // TODO enforce this once we can guarantee we always have schemas available + return this.getMissingSchemaPlaceholder(definitionReference); + } + if (!definition['$ref']) { + this.logger.debug('Reached a definition with no ref, terminating recursive lookup.'); + return definition; + } + const newRef: string = definition['$ref']; + if (visitedDefinitions.includes(newRef)) { + this.logger.warn('Circular reference detected. Terminating reference lookup. Visited definitions: ' + visitedDefinitions); + return definition; + } + const innerDef = this.getDefinitionRecursive(newRef, newSchemaId, visitedDefinitions); + return mergeSchemas(innerDef, definition); + } + + private getMissingSchemaPlaceholder(reference: string) { + return { + properties: { + 'missing-value': `MISSING OBJECT, ref: ${reference} could not be resolved` + } + }; + } + + /** + * + * @param definitionReference The reference to resolve. May be an absolute reference including a schema ID prefix, or a local reference. + * @returns The resolved object, or an empty object if the object could not be resolved. + */ + public getDefinition(definitionReference: string) { + this.logger.debug(`Resolving ${definitionReference} from schema directory.`); + return this.getDefinitionRecursive(definitionReference, 'pattern', []); + // TODO propagate the required fields + } + + /** + * Return the list of all loaded schemas. + */ + public getLoadedSchemas() { + return [...this.schemas.keys()]; + } + + /** + * Return the entire schema from the provided directory. + * @param schemaId The ID of the schema to load. + * @returns An entire schema as an object. + */ + public getSchema(schemaId: string) { + if (!this.schemas.has(schemaId)) { + const registered = this.getLoadedSchemas(); + this.logger.warn(`Schema with $id ${schemaId} not found. Returning empty object. Registered schemas: ${registered}`); + return undefined; + } + return this.schemas.get(schemaId); + } + + private async loadSchema(schemaPath: string) { + this.logger.debug('Loading ' + schemaPath); + const str = await readFile(schemaPath, 'utf-8'); + const parsed = JSON.parse(str); + const schemaId = parsed['$id']; + + if (!schemaId) { + this.logger.warn('Warning: bad schema found, no $id property was defined. Path: ', schemaPath); + return; + } + + if (!parsed['$schema']) { + this.logger.warn('Warning, loaded schema does not have $schema set and therefore may be invalid. Path: ', schemaPath); + } + + this.logger.debug('Loaded schema with $id: ' + schemaId); + + this.schemas.set(schemaId, parsed); + } + +} \ No newline at end of file diff --git a/cli/src/commands/generate/util.ts b/cli/src/commands/generate/util.ts new file mode 100644 index 00000000..f5918de9 --- /dev/null +++ b/cli/src/commands/generate/util.ts @@ -0,0 +1,12 @@ +import _ from 'lodash'; + +/** + * Recursively merge two schemas into a new object, without modifying either. + * In the event of a clash - i.e. two properties with the same name - the property from the second parameter will take precedence. + * @param s1 The first schema to merge + * @param s2 The second schema to merge. Takes precedence in the event of clashes. + * @returns A new merged schema + */ +export function mergeSchemas(s1: object, s2: object) { + return _.merge({}, s1, s2); +} \ No newline at end of file diff --git a/cli/src/commands/visualize/visualize.ts b/cli/src/commands/visualize/visualize.ts index cca21d46..846b1be9 100644 --- a/cli/src/commands/visualize/visualize.ts +++ b/cli/src/commands/visualize/visualize.ts @@ -4,6 +4,7 @@ import { initLogger } from '../helper.js'; import calmToDot from './calmToDot.js'; import { renderGraphFromSource } from 'graphviz-cli'; import { generate } from '../generate/generate.js'; +import { SchemaDirectory } from '../generate/schema-directory.js'; let logger: winston.Logger; @@ -35,7 +36,9 @@ export async function visualizeInstantiation(instantiationPath: string, output: export async function visualizePattern(patternPath: string, output: string, debug: boolean) { logger = initLogger(debug); - const instantiation = generate(patternPath, debug); + // TODO add a path to load schemas and generate intelligently + const schemaDir: SchemaDirectory = new SchemaDirectory(patternPath); + const instantiation = generate(patternPath, schemaDir, debug); logger.info('Generating an SVG from input'); diff --git a/cli/src/index.ts b/cli/src/index.ts index 86b70309..4e11ae6a 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -31,9 +31,10 @@ program .description('Generate an instantiation from a CALM pattern file.') .requiredOption('-p, --pattern ', 'Path to the pattern file to use. May be a file path or a URL.') .requiredOption('-o, --output ', 'Path location at which to output the generated file.') + .option('-s, --schemaDirectory ', 'Path to directory containing schemas to use in instantiation') .option('-v, --verbose', 'Enable verbose logging.', false) - .action((options) => { - runGenerate(options.pattern, options.output, !!options.verbose); + .action(async (options) => { + await runGenerate(options.pattern, options.output, options.schemaDirectory, !!options.verbose); }); program diff --git a/cli/test_fixtures/recursive_refs/recursive.json b/cli/test_fixtures/recursive_refs/recursive.json new file mode 100644 index 00000000..7843a8dc --- /dev/null +++ b/cli/test_fixtures/recursive_refs/recursive.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://raw.githubusercontent.com/finos-labs/architecture-as-code/main/calm/draft/2024-03/meta/calm.json", + "$id": "https://calm.com/recursive.json", + "title": "API Gateway Pattern", + "type": "object", + "$defs": { + "circular": { + "$ref": "https://calm.com/recursive.json#/$defs/circular", + "properties": { + "prop": { + "const": "test" + } + } + }, + "top-level": { + "$ref": "https://calm.com/recursive.json#/$defs/circular", + "properties": { + "top-level": { + "const": "test" + } + } + } + } +}