Skip to content

Commit

Permalink
Port json schema objects, asyncapi and openapi dereferencing (#5315)
Browse files Browse the repository at this point in the history
* Add test for json schema object and property level reference

* Add object and property level tests for json schema

* Add test with multiple references to a single file

* Port openapi dereference
  • Loading branch information
carlesarnal authored Oct 9, 2024
1 parent ef1c9cc commit 75c5c39
Show file tree
Hide file tree
Showing 17 changed files with 1,013 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,13 @@ $ curl -H "Authorization: Bearer $ACCESS_TOKEN" MY-REGISTRY-URL/apis/registry/v3
{"type":"record","name":"Item","namespace":"com.example.common","fields":[{"name":"itemId","type":{"type":"record","name":"ItemId","fields":[{"name":"id","type":"int"}]}}]}
----

#In Protobuf dereferencing content is only supported when all the schemas in the try belong to the same package.#
This support is currently implemented only for Avro, Protobuf, OpenAPI, AsyncAPI, and JSON Schema artifacts when the `dereference` parameter is specified in the API operation. This parameter is not supported for any other artifact types.

NOTE: For Protobuf artifacts, dereferencing content is supported only when all the schemas belong to the same package.

NOTE: Circular dependencies are allowed by some artifact types (e.g. JSON Schema) but are not supported by {registry}.

[role="_additional-resources"]
.Additional resources
* For more details, see the {registry-rest-api}.
* For more examples of artifact references, see the section on configuring each artifact type in {registry-client-serdes-config}.
* For more examples of artifact references, see the section on configuring each artifact type in {registry-client-serdes-config}.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import io.apicurio.datamodels.Library;
import io.apicurio.datamodels.TraverserDirection;
import io.apicurio.datamodels.models.Document;
import io.apicurio.datamodels.refs.IReferenceResolver;
import io.apicurio.registry.content.ContentHandle;
import io.apicurio.registry.content.TypedContent;
import io.apicurio.registry.content.util.ContentTypeUtil;
Expand All @@ -20,12 +19,11 @@ public class ApicurioDataModelsContentDereferencer implements ContentDereference
public TypedContent dereference(TypedContent content, Map<String, TypedContent> resolvedReferences) {
try {
JsonNode node = ContentTypeUtil.parseJsonOrYaml(content);
Document document = Library.readDocument((ObjectNode) node);
IReferenceResolver resolver = new RegistryReferenceResolver(resolvedReferences);
Document dereferencedDoc = Library.dereferenceDocument(document, resolver, false);
String dereferencedContentStr = Library.writeDocumentToJSONString(dereferencedDoc);
return TypedContent.create(ContentHandle.create(dereferencedContentStr),
ContentTypes.APPLICATION_JSON);
Document doc = Library.readDocument((ObjectNode) node);
ReferenceInliner inliner = new ReferenceInliner(resolvedReferences);
Library.visitTree(doc, inliner, TraverserDirection.down);
String dereferencedContent = Library.writeDocumentToJSONString(doc);
return TypedContent.create(ContentHandle.create(dereferencedContent), content.getContentType());
} catch (IOException e) {
throw new RuntimeException(e);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*
* Copyright 2023 Red Hat Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.apicurio.registry.content.dereference;

import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.apicurio.datamodels.Library;
import io.apicurio.datamodels.models.Node;
import io.apicurio.datamodels.models.Referenceable;
import io.apicurio.datamodels.models.visitors.AllNodeVisitor;
import io.apicurio.registry.content.ContentHandle;
import io.apicurio.registry.content.TypedContent;
import io.apicurio.registry.content.refs.JsonPointerExternalReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Map;

/**
* Inlines all references found in the data model.
*
* @author [email protected]
*/
public class ReferenceInliner extends AllNodeVisitor {
private static final Logger logger = LoggerFactory.getLogger(ReferenceInliner.class);
private static final ObjectMapper mapper = new ObjectMapper();
private final Map<String, TypedContent> resolvedReferences;

/**
* Constructor.
*
* @param resolvedReferences
*/
public ReferenceInliner(Map<String, TypedContent> resolvedReferences) {
this.resolvedReferences = resolvedReferences;
}

/**
* @see AllNodeVisitor#visitNode(Node)
*/
@Override
protected void visitNode(Node node) {
if (node instanceof Referenceable) {
String $ref = ((Referenceable) node).get$ref();
if ($ref != null && resolvedReferences.containsKey($ref)) {
inlineRef((Referenceable) node);
}
}
}

/**
* Inlines the given reference.
*
* @param refNode
*/
private void inlineRef(Referenceable refNode) {
String $ref = refNode.get$ref();

JsonPointerExternalReference refPointer = new JsonPointerExternalReference($ref);
ContentHandle refContent = resolvedReferences.get($ref).getContent();

// Get the specific node within the content that this $ref points to
ObjectNode refContentNode = getRefNodeFromContent(refContent, refPointer.getComponent());
if (refContentNode != null) {
// Read that content *into* the current node
Library.readNode(refContentNode, (Node) refNode);

// Set the $ref to null (now that we've inlined the referenced node)
refNode.set$ref(null);
}
}

private ObjectNode getRefNodeFromContent(ContentHandle refContent, String refComponent) {
try {
ObjectNode refContentRootNode = (ObjectNode) mapper.readTree(refContent.content());
if (refComponent != null) {
JsonPointer pointer = JsonPointer.compile(refComponent.substring(1));
JsonNode nodePointedTo = refContentRootNode.at(pointer);
if (!nodePointedTo.isMissingNode() && nodePointedTo.isObject()) {
return (ObjectNode) nodePointedTo;
}
} else {
return refContentRootNode;
}
} catch (Exception e) {
logger.error("Failed to get referenced node from $ref content.", e);
}
return null;
}

}
5 changes: 5 additions & 0 deletions schema-util/util-provider/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.apicurio</groupId>
<artifactId>apicurio-registry-utils-tests</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,16 @@
import io.apicurio.registry.content.refs.JsonSchemaReferenceFinder;
import io.apicurio.registry.content.refs.ReferenceFinder;
import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase;
import io.apicurio.registry.types.ContentTypes;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;

import static io.apicurio.registry.utils.tests.TestUtils.normalizeMultiLineString;

public class JsonSchemaContentDereferencerTest extends ArtifactUtilProviderTestBase {

@Test
Expand All @@ -30,4 +34,80 @@ public void testRewriteReferences() {
.contains(new JsonPointerExternalReference("https://www.example.org/schemas/ssn.json")));
}

@Test
public void testDereferenceObjectLevel() throws Exception {
TypedContent content = TypedContent.create(
resourceToContentHandle("json-schema-to-deref-object-level.json"),
ContentTypes.APPLICATION_JSON);
JsonSchemaDereferencer dereferencer = new JsonSchemaDereferencer();
// Note: order is important. The JSON schema dereferencer needs to convert the ContentHandle Map
// to a JSONSchema map. So it *must* resolve the leaves of the dependency tree before the branches.
Map<String, TypedContent> resolvedReferences = new LinkedHashMap<>();
resolvedReferences.put("types/city/qualification.json", TypedContent.create(
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("city/qualification.json", TypedContent.create(
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("identifier/qualification.json",
TypedContent.create(resourceToContentHandle("types/identifier/qualification.json"),
ContentTypes.APPLICATION_JSON));
resolvedReferences.put("types/all-types.json#/definitions/City", TypedContent
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("types/all-types.json#/definitions/Identifier", TypedContent
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
TypedContent modifiedContent = dereferencer.dereference(content, resolvedReferences);
String expectedContent = resourceToString("expected-testDereference-object-level-json.json");
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
normalizeMultiLineString(modifiedContent.getContent().content()));
}

@Test
public void testDereferencePropertyLevel() throws Exception {
TypedContent content = TypedContent.create(
resourceToContentHandle("json-schema-to-deref-property-level.json"),
ContentTypes.APPLICATION_JSON);
JsonSchemaDereferencer dereferencer = new JsonSchemaDereferencer();
// Note: order is important. The JSON schema dereferencer needs to convert the ContentHandle Map
// to a JSONSchema map. So it *must* resolve the leaves of the dependency tree before the branches.
Map<String, TypedContent> resolvedReferences = new LinkedHashMap<>();
resolvedReferences.put("types/city/qualification.json", TypedContent.create(
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("city/qualification.json", TypedContent.create(
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("identifier/qualification.json",
TypedContent.create(resourceToContentHandle("types/identifier/qualification.json"),
ContentTypes.APPLICATION_JSON));
resolvedReferences.put("types/all-types.json#/definitions/City/properties/name", TypedContent
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("types/all-types.json#/definitions/Identifier", TypedContent
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
TypedContent modifiedContent = dereferencer.dereference(content, resolvedReferences);
String expectedContent = resourceToString("expected-testDereference-property-level-json.json");
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
normalizeMultiLineString(modifiedContent.getContent().content()));
}

// Resolves multiple $refs using a single reference to a file with multiple definitions
@Test
public void testReferenceSingleFile() throws Exception {
TypedContent content = TypedContent.create(
resourceToContentHandle("json-schema-to-deref-property-level.json"),
ContentTypes.APPLICATION_JSON);
JsonSchemaDereferencer dereferencer = new JsonSchemaDereferencer();
// Note: order is important. The JSON schema dereferencer needs to convert the ContentHandle Map
// to a JSONSchema map. So it *must* resolve the leaves of the dependency tree before the branches.
Map<String, TypedContent> resolvedReferences = new LinkedHashMap<>();
resolvedReferences.put("types/city/qualification.json", TypedContent.create(
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("city/qualification.json", TypedContent.create(
resourceToContentHandle("types/city/qualification.json"), ContentTypes.APPLICATION_JSON));
resolvedReferences.put("identifier/qualification.json",
TypedContent.create(resourceToContentHandle("types/identifier/qualification.json"),
ContentTypes.APPLICATION_JSON));
resolvedReferences.put("types/all-types.json", TypedContent
.create(resourceToContentHandle("types/all-types.json"), ContentTypes.APPLICATION_JSON));
TypedContent modifiedContent = dereferencer.dereference(content, resolvedReferences);
String expectedContent = resourceToString("expected-testDereference-property-level-json.json");
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
normalizeMultiLineString(modifiedContent.getContent().content()));
}
}
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
package io.apicurio.registry.content.dereference;

import io.apicurio.registry.content.ContentHandle;
import io.apicurio.registry.content.TypedContent;
import io.apicurio.registry.content.refs.ExternalReference;
import io.apicurio.registry.content.refs.JsonPointerExternalReference;
import io.apicurio.registry.content.refs.OpenApiReferenceFinder;
import io.apicurio.registry.content.refs.ReferenceFinder;
import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase;
import io.apicurio.registry.types.ContentTypes;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.Map;
import java.util.Set;

import static io.apicurio.registry.utils.tests.TestUtils.normalizeMultiLineString;

public class OpenApiContentDereferencerTest extends ArtifactUtilProviderTestBase {

@Test
Expand All @@ -32,4 +36,22 @@ public void testRewriteReferences() {
"https://www.example.org/schemas/foo-types.json#/components/schemas/Foo")));
}

@Test
public void testDereference() throws Exception {
ContentHandle content = resourceToContentHandle("openapi-to-deref.json");
OpenApiDereferencer dereferencer = new OpenApiDereferencer();
Map<String, TypedContent> resolvedReferences = Map.of(
"http://types.example.org/all-types.json#/components/schemas/Foo",
TypedContent.create(resourceToContentHandle("all-types.json"), ContentTypes.APPLICATION_JSON),
"http://types.example.org/all-types.json#/components/schemas/Bar",
TypedContent.create(resourceToContentHandle("all-types.json"), ContentTypes.APPLICATION_JSON),
"http://types.example.org/address.json#/components/schemas/Address",
TypedContent.create(resourceToContentHandle("address.json"), ContentTypes.APPLICATION_JSON));
TypedContent modifiedContent = dereferencer
.dereference(TypedContent.create(content, ContentTypes.APPLICATION_JSON), resolvedReferences);
String expectedContent = resourceToString("expected-testDereference-openapi.json");
Assertions.assertEquals(normalizeMultiLineString(expectedContent),
normalizeMultiLineString(modifiedContent.getContent().content()));
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
{
"openapi": "3.0.2",
"info": {
"title": "address.json",
"version": "1.0.0",
"description": ""
},
"paths": {
"/": {}
},
"components": {
"schemas": {
"Address": {
"title": "Root Type for Address",
"description": "",
"type": "object",
"properties": {
"address1": {
"type": "string"
},
"city": {
"type": "string"
},
"state": {
"type": "string"
},
"zip": {
"type": "string"
}
},
"example": {
"address1": "225 West South St",
"city": "Springfield",
"state": "Massiginia",
"zip": "11556"
}
}
}
}
}
Loading

0 comments on commit 75c5c39

Please sign in to comment.