diff --git a/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/AbstractResource.java b/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/AbstractResource.java index 655c5debbd..25254bd8eb 100644 --- a/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/AbstractResource.java +++ b/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/AbstractResource.java @@ -25,6 +25,7 @@ import io.apicurio.registry.storage.error.ArtifactNotFoundException; import io.apicurio.registry.storage.error.RuleNotFoundException; import io.apicurio.registry.storage.error.VersionNotFoundException; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.types.Current; @@ -104,7 +105,8 @@ protected ArtifactVersionMetaDataDto createOrUpdateArtifact(String artifactId, S .map(dto -> ArtifactReference.builder().name(dto.getName()).groupId(dto.getGroupId()) .artifactId(dto.getArtifactId()).version(dto.getVersion()).build()) .collect(Collectors.toList()); - final Map resolvedReferences = storage.resolveReferences(parsedReferences); + final Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(parsedReferences, storage::getContentByReference); try { ContentHandle schemaContent; schemaContent = ContentHandle.create(schema); @@ -175,8 +177,9 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String artifac .getArtifactVersionContent(groupId, artifactId, version); TypedContent typedArtifactVersion = TypedContent .create(artifactVersion.getContent(), artifactVersion.getContentType()); - Map artifactVersionReferences = storage - .resolveReferences(artifactVersion.getReferences()); + Map artifactVersionReferences = RegistryContentUtils + .recursivelyResolveReferences(artifactVersion.getReferences(), + storage::getContentByReference); String dereferencedExistingContentSha = DigestUtils .sha256Hex(artifactTypeProvider.getContentDereferencer() .dereference(typedArtifactVersion, artifactVersionReferences) @@ -215,7 +218,8 @@ protected Map resolveReferences(List refe return artifactReferenceDto; }).collect(Collectors.toList()); - resolvedReferences = storage.resolveReferences(referencesAsDtos); + resolvedReferences = RegistryContentUtils.recursivelyResolveReferences(referencesAsDtos, + storage::getContentByReference); if (references.size() > resolvedReferences.size()) { // There are unresolvable references, which is not allowed. diff --git a/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SchemasResourceImpl.java b/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SchemasResourceImpl.java index 0fac97e67b..b3c39b5216 100644 --- a/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SchemasResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/ccompat/rest/v7/impl/SchemasResourceImpl.java @@ -15,6 +15,7 @@ import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto; import io.apicurio.registry.storage.dto.ContentWrapperDto; import io.apicurio.registry.storage.dto.StoredArtifactVersionDto; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.types.VersionState; import io.apicurio.registry.util.ArtifactTypeUtil; @@ -47,8 +48,10 @@ public SchemaInfo getSchema(int id, String subject, String groupId) { references = contentWrapper.getReferences(); } TypedContent typedContent = TypedContent.create(contentHandle, contentType); - return converter.convert(contentHandle, ArtifactTypeUtil.determineArtifactType(typedContent, null, - storage.resolveReferences(references), factory), references); + return converter.convert(contentHandle, + ArtifactTypeUtil.determineArtifactType(typedContent, null, RegistryContentUtils + .recursivelyResolveReferences(references, storage::getContentByReference), factory), + references); } @Override diff --git a/app/src/main/java/io/apicurio/registry/rest/v2/AbstractResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v2/AbstractResourceImpl.java index fd405a9fb6..32ac42acca 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v2/AbstractResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v2/AbstractResourceImpl.java @@ -1,143 +1,10 @@ package io.apicurio.registry.rest.v2; -import io.apicurio.common.apps.config.Info; -import io.apicurio.registry.content.TypedContent; -import io.apicurio.registry.content.dereference.ContentDereferencer; -import io.apicurio.registry.content.refs.JsonPointerExternalReference; -import io.apicurio.registry.storage.RegistryStorage; -import io.apicurio.registry.storage.dto.ArtifactReferenceDto; -import io.apicurio.registry.types.Current; -import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider; -import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory; -import io.apicurio.registry.utils.StringUtil; -import jakarta.inject.Inject; import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.core.Context; -import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.slf4j.Logger; - -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; public abstract class AbstractResourceImpl { - @Inject - Logger log; - - @Inject - @Current - RegistryStorage storage; - - @Inject - ArtifactTypeUtilProviderFactory factory; - @Context HttpServletRequest request; - - @ConfigProperty(name = "apicurio.apis.v2.base-href", defaultValue = "_") - @Info(category = "api", description = "API base href (URI)", availableSince = "2.5.0.Final") - String apiBaseHref; - - /** - * Handle the content references based on the value of "dereference" - this can mean we need to fully - * dereference the content. - * - * @param dereference - * @param content - */ - protected TypedContent handleContentReferences(boolean dereference, String artifactType, - TypedContent content, List references) { - // Dereference or rewrite references - if (!references.isEmpty() && dereference) { - ArtifactTypeUtilProvider artifactTypeProvider = factory.getArtifactTypeProvider(artifactType); - ContentDereferencer contentDereferencer = artifactTypeProvider.getContentDereferencer(); - Map resolvedReferences = storage.resolveReferences(references); - content = contentDereferencer.dereference(content, resolvedReferences); - } - return content; - } - - /** - * Convert the list of references into a list of REST API URLs that point to the content. This means that - * we generate a REST API URL from the GAV (groupId, artifactId, version) information found in each - * reference. - * - * @param references - */ - protected Map resolveReferenceUrls(List references) { - Map rval = new HashMap<>(); - for (ArtifactReferenceDto reference : references) { - String resolvedReferenceUrl = resolveReferenceUrl(reference); - if (reference.getName().contains("#")) { - JsonPointerExternalReference jpRef = new JsonPointerExternalReference(reference.getName()); - resolvedReferenceUrl = resolvedReferenceUrl + jpRef.getComponent(); - } - if (resolvedReferenceUrl != null) { - rval.put(reference.getName(), resolvedReferenceUrl); - } - } - return rval; - } - - /** - * Convert a single artifact reference to a REST API URL. This means that we generate a REST API URL from - * the GAV (groupId, artifactId, version) information found in the reference. - * - * @param reference - */ - protected String resolveReferenceUrl(ArtifactReferenceDto reference) { - URI baseHref = null; - try { - if (!"_".equals(apiBaseHref)) { - baseHref = new URI(apiBaseHref); - } else { - baseHref = getApiBaseHrefFromXForwarded(request); - if (baseHref == null) { - baseHref = getApiBaseHrefFromRequest(request); - } - } - } catch (URISyntaxException e) { - this.log.error("Error trying to determine the baseHref of the REST API.", e); - return null; - } - - if (baseHref == null) { - this.log.warn("Failed to determine baseHref for the REST API."); - return null; - } - - String path = String.format("/apis/registry/v2/groups/%s/artifacts/%s/versions/%s?references=REWRITE", - URLEncoder.encode(reference.getGroupId(), StandardCharsets.UTF_8), - URLEncoder.encode(reference.getArtifactId(), StandardCharsets.UTF_8), - URLEncoder.encode(reference.getVersion(), StandardCharsets.UTF_8)); - return baseHref.resolve(path).toString(); - } - - /** - * Resolves a host name from the information found in X-Forwarded-Host and X-Forwarded-Proto. - */ - private static URI getApiBaseHrefFromXForwarded(HttpServletRequest request) throws URISyntaxException { - String fproto = request.getHeader("X-Forwarded-Proto"); - String fhost = request.getHeader("X-Forwarded-Host"); - if (!StringUtil.isEmpty(fproto) && !StringUtil.isEmpty(fhost)) { - return new URI(fproto + "://" + fhost); - } else { - return null; - } - } - - /** - * Resolves a host name from the request information. - */ - private static URI getApiBaseHrefFromRequest(HttpServletRequest request) throws URISyntaxException { - String requestUrl = request.getRequestURL().toString(); - URI requestUri = new URI(requestUrl); - return requestUri.resolve("/"); - } - } diff --git a/app/src/main/java/io/apicurio/registry/rest/v2/GroupsResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v2/GroupsResourceImpl.java index eecd5fac1e..4751ae1032 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v2/GroupsResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v2/GroupsResourceImpl.java @@ -38,9 +38,9 @@ import io.apicurio.registry.rest.v2.beans.UpdateState; import io.apicurio.registry.rest.v2.beans.VersionMetaData; import io.apicurio.registry.rest.v2.beans.VersionSearchResults; -import io.apicurio.registry.rest.v2.shared.CommonResourceOperations; import io.apicurio.registry.rules.RuleApplicationType; import io.apicurio.registry.rules.RulesService; +import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.storage.RegistryStorage.RetrievalBehavior; import io.apicurio.registry.storage.dto.ArtifactMetaDataDto; import io.apicurio.registry.storage.dto.ArtifactReferenceDto; @@ -63,12 +63,15 @@ import io.apicurio.registry.storage.error.InvalidArtifactIdException; import io.apicurio.registry.storage.error.InvalidGroupIdException; import io.apicurio.registry.storage.error.VersionNotFoundException; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.ArtifactState; import io.apicurio.registry.types.ContentTypes; +import io.apicurio.registry.types.Current; import io.apicurio.registry.types.ReferenceType; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.types.VersionState; import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider; +import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory; import io.apicurio.registry.util.ArtifactIdGenerator; import io.apicurio.registry.util.ArtifactTypeUtil; import io.apicurio.registry.utils.ArtifactIdValidator; @@ -78,10 +81,12 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.interceptor.Interceptors; +import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.HttpMethod; import jakarta.ws.rs.NotAllowedException; import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; import org.apache.commons.lang3.tuple.Pair; import org.jose4j.base64url.Base64; @@ -130,7 +135,7 @@ @ApplicationScoped @Interceptors({ ResponseErrorLivenessCheck.class, ResponseTimeoutReadinessCheck.class }) @Logged -public class GroupsResourceImpl extends AbstractResourceImpl implements GroupsResource { +public class GroupsResourceImpl implements GroupsResource { private static final String EMPTY_CONTENT_ERROR_MESSAGE = "Empty content is not allowed."; @SuppressWarnings("unused") @@ -149,11 +154,18 @@ public class GroupsResourceImpl extends AbstractResourceImpl implements GroupsRe SecurityIdentity securityIdentity; @Inject - CommonResourceOperations common; + @Current + RegistryStorage storage; + + @Inject + ArtifactTypeUtilProviderFactory factory; @Inject io.apicurio.registry.rest.v3.GroupsResourceImpl v3; + @Context + HttpServletRequest request; + /** * @see io.apicurio.registry.rest.v2.GroupsResource#getLatestArtifact(java.lang.String, java.lang.String, * Boolean) @@ -179,8 +191,25 @@ public Response getLatestArtifact(String groupId, String artifactId, Boolean der TypedContent contentToReturn = TypedContent.create(artifact.getContent(), artifact.getContentType()); - contentToReturn = handleContentReferences(dereference, metaData.getArtifactType(), - contentToReturn, artifact.getReferences()); + + ArtifactTypeUtilProvider artifactTypeProvider = factory + .getArtifactTypeProvider(metaData.getArtifactType()); + + if (dereference && !artifact.getReferences().isEmpty()) { + if (artifactTypeProvider.supportsReferencesWithContext()) { + RegistryContentUtils.RewrittenContentHolder rewrittenContent = RegistryContentUtils + .recursivelyResolveReferencesWithContext(contentToReturn, + metaData.getArtifactType(), artifact.getReferences(), + storage::getContentByReference); + + contentToReturn = artifactTypeProvider.getContentDereferencer().dereference( + rewrittenContent.getRewrittenContent(), rewrittenContent.getResolvedReferences()); + } else { + contentToReturn = artifactTypeProvider.getContentDereferencer() + .dereference(contentToReturn, RegistryContentUtils.recursivelyResolveReferences( + artifact.getReferences(), storage::getContentByReference)); + } + } Response.ResponseBuilder builder = Response.ok(contentToReturn.getContent(), contentToReturn.getContentType()); @@ -642,8 +671,24 @@ public Response getArtifactVersion(String groupId, String artifactId, String ver artifactId, version); TypedContent contentToReturn = TypedContent.create(artifact.getContent(), artifact.getContentType()); - contentToReturn = handleContentReferences(dereference, metaData.getArtifactType(), contentToReturn, - artifact.getReferences()); + + ArtifactTypeUtilProvider artifactTypeProvider = factory + .getArtifactTypeProvider(metaData.getArtifactType()); + + if (dereference && !artifact.getReferences().isEmpty()) { + if (artifactTypeProvider.supportsReferencesWithContext()) { + RegistryContentUtils.RewrittenContentHolder rewrittenContent = RegistryContentUtils + .recursivelyResolveReferencesWithContext(contentToReturn, metaData.getArtifactType(), + artifact.getReferences(), storage::getContentByReference); + + contentToReturn = artifactTypeProvider.getContentDereferencer().dereference( + rewrittenContent.getRewrittenContent(), rewrittenContent.getResolvedReferences()); + } else { + contentToReturn = artifactTypeProvider.getContentDereferencer().dereference(contentToReturn, + RegistryContentUtils.recursivelyResolveReferences(artifact.getReferences(), + storage::getContentByReference)); + } + } Response.ResponseBuilder builder = Response.ok(contentToReturn.getContent(), contentToReturn.getContentType()); @@ -1064,7 +1109,8 @@ private ArtifactMetaData createArtifactWithRefs(String groupId, String xRegistry final List referencesAsDtos = toReferenceDtos(references); // Try to resolve the new artifact references and the nested ones (if any) - final Map resolvedReferences = storage.resolveReferences(referencesAsDtos); + final Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference); rulesService.applyRules(defaultGroupIdToNull(groupId), artifactId, artifactType, typedContent, RuleApplicationType.CREATE, toV3Refs(references), resolvedReferences); @@ -1200,7 +1246,8 @@ private VersionMetaData createArtifactVersionWithRefs(String groupId, String art final List referencesAsDtos = toReferenceDtos(references); // Try to resolve the new artifact references and the nested ones (if any) - final Map resolvedReferences = storage.resolveReferences(referencesAsDtos); + final Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference); String artifactType = lookupArtifactType(groupId, artifactId); TypedContent typedContent = TypedContent.create(content, ct); @@ -1325,7 +1372,8 @@ private ArtifactMetaData updateArtifactInternal(String groupId, String artifactI // passed references does not exist. final List referencesAsDtos = toReferenceDtos(references); - final Map resolvedReferences = storage.resolveReferences(referencesAsDtos); + final Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference); TypedContent typedContent = TypedContent.create(content, contentType); rulesService.applyRules(defaultGroupIdToNull(groupId), artifactId, artifactType, typedContent, diff --git a/app/src/main/java/io/apicurio/registry/rest/v2/IdsResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v2/IdsResourceImpl.java index d8d810b93e..584f5d1ffb 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v2/IdsResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v2/IdsResourceImpl.java @@ -11,14 +11,19 @@ import io.apicurio.registry.rest.HeadersHack; import io.apicurio.registry.rest.v2.beans.ArtifactReference; import io.apicurio.registry.rest.v2.shared.CommonResourceOperations; +import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto; import io.apicurio.registry.storage.dto.ContentWrapperDto; import io.apicurio.registry.storage.dto.StoredArtifactVersionDto; import io.apicurio.registry.storage.error.ArtifactNotFoundException; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.ArtifactMediaTypes; import io.apicurio.registry.types.ArtifactState; +import io.apicurio.registry.types.Current; import io.apicurio.registry.types.ReferenceType; import io.apicurio.registry.types.VersionState; +import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider; +import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.interceptor.Interceptors; @@ -31,11 +36,18 @@ @ApplicationScoped @Interceptors({ ResponseErrorLivenessCheck.class, ResponseTimeoutReadinessCheck.class }) @Logged -public class IdsResourceImpl extends AbstractResourceImpl implements IdsResource { +public class IdsResourceImpl implements IdsResource { @Inject CommonResourceOperations common; + @Inject + @Current + RegistryStorage storage; + + @Inject + ArtifactTypeUtilProviderFactory factory; + private void checkIfDeprecated(Supplier stateSupplier, String artifactId, String version, Response.ResponseBuilder builder) { HeadersHack.checkIfDeprecated(stateSupplier, null, artifactId, version, builder); @@ -70,8 +82,24 @@ public Response getContentByGlobalId(long globalId, Boolean dereference) { StoredArtifactVersionDto artifact = storage.getArtifactVersionContent(globalId); TypedContent contentToReturn = TypedContent.create(artifact.getContent(), artifact.getContentType()); - handleContentReferences(dereference, metaData.getArtifactType(), contentToReturn, - artifact.getReferences()); + + ArtifactTypeUtilProvider artifactTypeProvider = factory + .getArtifactTypeProvider(metaData.getArtifactType()); + + if (dereference && !artifact.getReferences().isEmpty()) { + if (artifactTypeProvider.supportsReferencesWithContext()) { + RegistryContentUtils.RewrittenContentHolder rewrittenContent = RegistryContentUtils + .recursivelyResolveReferencesWithContext(contentToReturn, metaData.getArtifactType(), + artifact.getReferences(), storage::getContentByReference); + + contentToReturn = artifactTypeProvider.getContentDereferencer().dereference( + rewrittenContent.getRewrittenContent(), rewrittenContent.getResolvedReferences()); + } else { + contentToReturn = artifactTypeProvider.getContentDereferencer().dereference(contentToReturn, + RegistryContentUtils.recursivelyResolveReferences(artifact.getReferences(), + storage::getContentByReference)); + } + } Response.ResponseBuilder builder = Response.ok(contentToReturn.getContent(), contentToReturn.getContentType()); diff --git a/app/src/main/java/io/apicurio/registry/rest/v3/AbstractResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v3/AbstractResourceImpl.java index 629cccd375..63399b58c0 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v3/AbstractResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v3/AbstractResourceImpl.java @@ -7,6 +7,7 @@ import io.apicurio.registry.rest.v3.beans.HandleReferencesType; import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.storage.dto.ArtifactReferenceDto; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.Current; import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider; import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory; @@ -50,13 +51,22 @@ public abstract class AbstractResourceImpl { */ protected TypedContent handleContentReferences(HandleReferencesType referencesType, String artifactType, TypedContent content, List references) { - // Dereference or rewrite references if (!references.isEmpty()) { if (referencesType == HandleReferencesType.DEREFERENCE) { ArtifactTypeUtilProvider artifactTypeProvider = factory.getArtifactTypeProvider(artifactType); - ContentDereferencer contentDereferencer = artifactTypeProvider.getContentDereferencer(); - Map resolvedReferences = storage.resolveReferences(references); - content = contentDereferencer.dereference(content, resolvedReferences); + + if (artifactTypeProvider.supportsReferencesWithContext()) { + RegistryContentUtils.RewrittenContentHolder rewrittenContent = RegistryContentUtils + .recursivelyResolveReferencesWithContext(content, artifactType, references, + storage::getContentByReference); + + content = artifactTypeProvider.getContentDereferencer().dereference( + rewrittenContent.getRewrittenContent(), rewrittenContent.getResolvedReferences()); + } else { + content = artifactTypeProvider.getContentDereferencer().dereference(content, + RegistryContentUtils.recursivelyResolveReferences(references, + storage::getContentByReference)); + } } else if (referencesType == HandleReferencesType.REWRITE) { ArtifactTypeUtilProvider artifactTypeProvider = factory.getArtifactTypeProvider(artifactType); ContentDereferencer contentDereferencer = artifactTypeProvider.getContentDereferencer(); @@ -71,7 +81,7 @@ protected TypedContent handleContentReferences(HandleReferencesType referencesTy * Convert the list of references into a list of REST API URLs that point to the content. This means that * we generate a REST API URL from the GAV (groupId, artifactId, version) information found in each * reference. - * + * * @param references */ protected Map resolveReferenceUrls(List references) { @@ -92,7 +102,7 @@ protected Map resolveReferenceUrls(List re /** * Convert a single artifact reference to a REST API URL. This means that we generate a REST API URL from * the GAV (groupId, artifactId, version) information found in the reference. - * + * * @param reference */ protected String resolveReferenceUrl(ArtifactReferenceDto reference) { diff --git a/app/src/main/java/io/apicurio/registry/rest/v3/GroupsResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v3/GroupsResourceImpl.java index 09925dea76..fbacea92be 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v3/GroupsResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v3/GroupsResourceImpl.java @@ -77,6 +77,7 @@ import io.apicurio.registry.storage.error.InvalidArtifactIdException; import io.apicurio.registry.storage.error.InvalidGroupIdException; import io.apicurio.registry.storage.error.VersionNotFoundException; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.ReferenceType; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.types.VersionState; @@ -796,7 +797,8 @@ public CreateArtifactResponse createArtifact(String groupId, IfArtifactExists if final List referencesAsDtos = toReferenceDtos(references); // Try to resolve the references - final Map resolvedReferences = storage.resolveReferences(referencesAsDtos); + final Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference); // Apply any configured rules if (content != null) { @@ -896,7 +898,8 @@ public VersionMetaData createArtifactVersion(String groupId, String artifactId, data.getContent().getReferences()); // Try to resolve the new artifact references and the nested ones (if any) - final Map resolvedReferences = storage.resolveReferences(referencesAsDtos); + final Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference); String artifactType = lookupArtifactType(groupId, artifactId); TypedContent typedContent = TypedContent.create(content, ct); @@ -1167,7 +1170,8 @@ private CreateArtifactResponse updateArtifactInternal(String groupId, String art // passed references does not exist. final List referencesAsDtos = toReferenceDtos(references); - final Map resolvedReferences = storage.resolveReferences(referencesAsDtos); + final Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference); final TypedContent typedContent = TypedContent.create(content, contentType); rulesService.applyRules(new GroupId(groupId).getRawGroupIdWithNull(), artifactId, artifactType, typedContent, RuleApplicationType.UPDATE, references, resolvedReferences); diff --git a/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java index 4872e450e8..1a50d5f334 100644 --- a/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java @@ -56,7 +56,6 @@ import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -771,11 +770,7 @@ void createRoleMapping(String principalId, String role, String principalName) */ List getStaleConfigProperties(Instant since); - /** - * @return The artifact references resolved as a map containing the reference name as key and the - * referenced artifact content. - */ - Map resolveReferences(List references); + ContentWrapperDto getContentByReference(ArtifactReferenceDto reference); /** * Quickly checks for the existence of a given artifact. diff --git a/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java b/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java index b2a2d9800e..4040f0d6ba 100644 --- a/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java +++ b/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java @@ -7,18 +7,7 @@ import io.apicurio.registry.model.GA; import io.apicurio.registry.model.VersionId; import io.apicurio.registry.storage.RegistryStorage; -import io.apicurio.registry.storage.dto.ArtifactMetaDataDto; -import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto; -import io.apicurio.registry.storage.dto.BranchMetaDataDto; -import io.apicurio.registry.storage.dto.CommentDto; -import io.apicurio.registry.storage.dto.ContentWrapperDto; -import io.apicurio.registry.storage.dto.DownloadContextDto; -import io.apicurio.registry.storage.dto.EditableArtifactMetaDataDto; -import io.apicurio.registry.storage.dto.EditableBranchMetaDataDto; -import io.apicurio.registry.storage.dto.EditableGroupMetaDataDto; -import io.apicurio.registry.storage.dto.EditableVersionMetaDataDto; -import io.apicurio.registry.storage.dto.GroupMetaDataDto; -import io.apicurio.registry.storage.dto.RuleConfigurationDto; +import io.apicurio.registry.storage.dto.*; import io.apicurio.registry.storage.error.ArtifactNotFoundException; import io.apicurio.registry.storage.error.GroupAlreadyExistsException; import io.apicurio.registry.storage.error.GroupNotFoundException; @@ -469,4 +458,10 @@ public String createSnapshot(String snapshotLocation) throws RegistryStorageExce checkReadOnly(); return delegate.createSnapshot(snapshotLocation); } + + @Override + public ContentWrapperDto getContentByReference(ArtifactReferenceDto reference) { + checkReadOnly(); + return delegate.getContentByReference(reference); + } } diff --git a/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java b/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java index 275598ab41..c981d522a2 100644 --- a/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java +++ b/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java @@ -35,7 +35,6 @@ import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -272,11 +271,6 @@ public DynamicConfigPropertyDto getRawConfigProperty(String propertyName) { return delegate.getRawConfigProperty(propertyName); } - @Override - public Map resolveReferences(List references) { - return delegate.resolveReferences(references); - } - @Override public boolean isEmpty() { return delegate.isEmpty(); @@ -353,6 +347,11 @@ public Optional contentIdFromHash(String contentHash) { return delegate.contentIdFromHash(contentHash); } + @Override + public ContentWrapperDto getContentByReference(ArtifactReferenceDto reference) { + return delegate.getContentByReference(reference); + } + @Override public List getEnabledArtifactContentIds(String groupId, String artifactId) { return delegate.getEnabledArtifactContentIds(groupId, artifactId); diff --git a/app/src/main/java/io/apicurio/registry/storage/dto/ContentWrapperDto.java b/app/src/main/java/io/apicurio/registry/storage/dto/ContentWrapperDto.java index 5515535eb8..e1da307bad 100644 --- a/app/src/main/java/io/apicurio/registry/storage/dto/ContentWrapperDto.java +++ b/app/src/main/java/io/apicurio/registry/storage/dto/ContentWrapperDto.java @@ -17,4 +17,5 @@ public class ContentWrapperDto { private String contentType; private ContentHandle content; private List references; + private String artifactType; } diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java index 2aa7a87da4..ccdd964f8a 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java @@ -43,7 +43,6 @@ import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -391,6 +390,11 @@ public List getStaleConfigProperties(Instant since) { return proxy(storage -> storage.getStaleConfigProperties(since)); } + @Override + public ContentWrapperDto getContentByReference(ArtifactReferenceDto reference) { + return proxy(storage -> storage.getContentByReference(reference)); + } + @Override public boolean isContentExists(String contentHash) { return proxy(storage -> storage.isContentExists(contentHash)); @@ -411,11 +415,6 @@ public boolean isRoleMappingExists(String principalId) { return proxy(storage -> storage.isRoleMappingExists(principalId)); } - @Override - public Map resolveReferences(List references) { - return proxy(storage -> storage.resolveReferences(references)); - } - @Override public Optional contentIdFromHash(String contentHash) { return proxy(storage -> storage.contentIdFromHash(contentHash)); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java index 397b7d2c43..9c7cb3b521 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java @@ -735,6 +735,11 @@ public void deleteAllExpiredDownloads() throws RegistryStorageException { coordinator.waitForResponse(uuid); } + @Override + public ContentWrapperDto getContentByReference(ArtifactReferenceDto reference) { + return sqlStore.getContentByReference(reference); + } + /** * @see io.apicurio.registry.storage.RegistryStorage#createArtifactVersionComment(java.lang.String, * java.lang.String, java.lang.String, java.lang.String) diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java index cdc8ac8983..3f077b56c2 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java @@ -115,6 +115,7 @@ import io.quarkus.security.identity.SecurityIdentity; import jakarta.enterprise.event.Event; import jakarta.inject.Inject; +import jakarta.transaction.Transactional; import jakarta.validation.ValidationException; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -142,8 +143,8 @@ import java.util.function.Function; import java.util.stream.Stream; +import static io.apicurio.registry.storage.impl.sql.RegistryContentUtils.normalizeGroupId; import static io.apicurio.registry.storage.impl.sql.RegistryStorageContentUtils.notEmpty; -import static io.apicurio.registry.storage.impl.sql.SqlUtil.normalizeGroupId; import static io.apicurio.registry.utils.StringUtil.asLowerCase; import static io.apicurio.registry.utils.StringUtil.limitStr; import static java.util.stream.Collectors.toList; @@ -517,7 +518,7 @@ public Pair createArtifact(Stri } Map labels = amd.getLabels(); - String labelsStr = SqlUtil.serializeLabels(labels); + String labelsStr = RegistryContentUtils.serializeLabels(labels); // Create a row in the artifacts table. handle.createUpdate(sqlStatements.insertArtifact()).bind(0, normalizeGroupId(groupId)) @@ -570,7 +571,7 @@ private ArtifactVersionMetaDataDto createArtifactVersionRaw(Handle handle, boole } ArtifactState state = ArtifactState.ENABLED; - String labelsStr = SqlUtil.serializeLabels(metaData.getLabels()); + String labelsStr = RegistryContentUtils.serializeLabels(metaData.getLabels()); Long globalId = nextGlobalIdRaw(handle); GAV gav; @@ -700,7 +701,7 @@ private Long ensureContentAndGetId(String artifactType, ContentWrapperDto conten contentHash = utils.getContentHash(content, references); canonicalContentHash = utils.getCanonicalContentHash(content, artifactType, references, referenceResolver); - serializedReferences = SqlUtil.serializeReferences(references); + serializedReferences = RegistryContentUtils.serializeReferences(references); } else { contentHash = utils.getContentHash(content, null); canonicalContentHash = utils.getCanonicalContentHash(content, artifactType, null, null); @@ -1168,7 +1169,7 @@ public void updateArtifactMetaData(String groupId, String artifactId, // Update labels if (metaData.getLabels() != null) { int rowCount = handle.createUpdate(sqlStatements.updateArtifactLabels()) - .bind(0, SqlUtil.serializeLabels(metaData.getLabels())) + .bind(0, RegistryContentUtils.serializeLabels(metaData.getLabels())) .bind(1, normalizeGroupId(groupId)).bind(2, artifactId).execute(); modified = true; if (rowCount == 0) { @@ -1768,7 +1769,7 @@ public void updateArtifactVersionMetaData(String groupId, String artifactId, Str if (editableMetadata.getLabels() != null) { int rowCount = handle.createUpdate(sqlStatements.updateArtifactVersionLabelsByGAV()) - .bind(0, SqlUtil.serializeLabels(editableMetadata.getLabels())) + .bind(0, RegistryContentUtils.serializeLabels(editableMetadata.getLabels())) .bind(1, normalizeGroupId(groupId)).bind(2, artifactId).bind(3, version).execute(); if (rowCount == 0) { throw new VersionNotFoundException(groupId, artifactId, version); @@ -2056,7 +2057,7 @@ public void createGroup(GroupMetaDataDto group) .bind(4, group.getCreatedOn() == 0 ? new Date() : new Date(group.getCreatedOn())) .bind(5, group.getModifiedBy()) .bind(6, group.getModifiedOn() == 0 ? new Date() : new Date(group.getModifiedOn())) - .bind(7, SqlUtil.serializeLabels(group.getLabels())).execute(); + .bind(7, RegistryContentUtils.serializeLabels(group.getLabels())).execute(); // Insert new labels into the "group_labels" table Map labels = group.getLabels(); @@ -2118,8 +2119,9 @@ public void updateGroupMetaData(String groupId, EditableGroupMetaDataDto dto) { handles.withHandleNoException(handle -> { // Update the row in the groups table int rows = handle.createUpdate(sqlStatements.updateGroup()).bind(0, dto.getDescription()) - .bind(1, modifiedBy).bind(2, modifiedOn).bind(3, SqlUtil.serializeLabels(dto.getLabels())) - .bind(4, groupId).execute(); + .bind(1, modifiedBy).bind(2, modifiedOn) + .bind(3, RegistryContentUtils.serializeLabels(dto.getLabels())).bind(4, groupId) + .execute(); if (rows == 0) { throw new GroupNotFoundException(groupId); } @@ -2508,13 +2510,6 @@ public void deleteAllUserData() { } - @Override - public Map resolveReferences(List references) { - return handles.withHandleNoException(handle -> { - return resolveReferencesRaw(handle, references); - }); - } - private Map resolveReferencesRaw(Handle handle, List references) { if (references == null || references.isEmpty()) { @@ -2713,6 +2708,20 @@ public GroupSearchResultsDto searchGroups(Set filters, OrderBy ord }); } + @Override + @Transactional + public ContentWrapperDto getContentByReference(ArtifactReferenceDto reference) { + try { + var meta = getArtifactVersionMetaData(reference.getGroupId(), reference.getArtifactId(), + reference.getVersion()); + ContentWrapperDto artifactByContentId = getContentById(meta.getContentId()); + artifactByContentId.setArtifactType(meta.getArtifactType()); + return artifactByContentId; + } catch (VersionNotFoundException e) { + return null; + } + } + private void resolveReferencesRaw(Handle handle, Map resolvedReferences, List references) { if (references != null && !references.isEmpty()) { @@ -2866,7 +2875,7 @@ public void importArtifactRule(ArtifactRuleEntity entity) { public void importArtifact(ArtifactEntity entity) { handles.withHandleNoException(handle -> { if (!isArtifactExistsRaw(handle, entity.groupId, entity.artifactId)) { - String labelsStr = SqlUtil.serializeLabels(entity.labels); + String labelsStr = RegistryContentUtils.serializeLabels(entity.labels); handle.createUpdate(sqlStatements.insertArtifact()).bind(0, normalizeGroupId(entity.groupId)) .bind(1, entity.artifactId).bind(2, entity.artifactType).bind(3, entity.owner) .bind(4, new Date(entity.createdOn)).bind(5, entity.modifiedBy) @@ -2905,8 +2914,8 @@ public void importArtifactVersion(ArtifactVersionEntity entity) { .bind(6, entity.name).bind(7, entity.description).bind(8, entity.owner) .bind(9, new Date(entity.createdOn)).bind(10, entity.modifiedBy) .bind(11, new Date(entity.modifiedOn)) - .bind(12, SqlUtil.serializeLabels(entity.labels)).bind(13, entity.contentId) - .execute(); + .bind(12, RegistryContentUtils.serializeLabels(entity.labels)) + .bind(13, entity.contentId).execute(); // Insert labels into the "version_labels" table if (entity.labels != null && !entity.labels.isEmpty()) { @@ -2934,7 +2943,7 @@ public void importContent(ContentEntity entity) { .bind(4, entity.contentBytes).bind(5, entity.serializedReferences).execute(); insertReferencesRaw(handle, entity.contentId, - SqlUtil.deserializeReferences(entity.serializedReferences)); + RegistryContentUtils.deserializeReferences(entity.serializedReferences)); } else { throw new ContentAlreadyExistsException(entity.contentId); } @@ -2958,11 +2967,12 @@ public void importGroup(GroupEntity entity) { throw new GroupAlreadyExistsException(entity.groupId); } - handle.createUpdate(sqlStatements.importGroup()).bind(0, SqlUtil.normalizeGroupId(entity.groupId)) + handle.createUpdate(sqlStatements.importGroup()) + .bind(0, RegistryContentUtils.normalizeGroupId(entity.groupId)) .bind(1, entity.description).bind(2, entity.artifactsType).bind(3, entity.owner) .bind(4, new Date(entity.createdOn)).bind(5, entity.modifiedBy) - .bind(6, new Date(entity.modifiedOn)).bind(7, SqlUtil.serializeLabels(entity.labels)) - .execute(); + .bind(6, new Date(entity.modifiedOn)) + .bind(7, RegistryContentUtils.serializeLabels(entity.labels)).execute(); // Insert labels into the "group_labels" table if (entity.labels != null && !entity.labels.isEmpty()) { diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/RegistryContentUtils.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/RegistryContentUtils.java new file mode 100644 index 0000000000..f7f5db6fef --- /dev/null +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/RegistryContentUtils.java @@ -0,0 +1,356 @@ +package io.apicurio.registry.storage.impl.sql; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.content.TypedContent; +import io.apicurio.registry.content.refs.JsonPointerExternalReference; +import io.apicurio.registry.storage.dto.ArtifactReferenceDto; +import io.apicurio.registry.storage.dto.ContentWrapperDto; +import io.apicurio.registry.types.RegistryException; +import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider; +import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory; +import io.apicurio.registry.types.provider.DefaultArtifactTypeUtilProviderImpl; +import io.apicurio.registry.utils.StringUtil; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class RegistryContentUtils { + + private static final Logger log = LoggerFactory.getLogger(RegistryContentUtils.class); + + private static final String NULL_GROUP_ID = "__$GROUPID$__"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public static final ArtifactTypeUtilProviderFactory ARTIFACT_TYPE_UTIL = new DefaultArtifactTypeUtilProviderImpl(); + + private RegistryContentUtils() { + } + + /** + * Recursively resolve the references. + */ + public static Map recursivelyResolveReferences( + List references, Function loader) { + if (references == null || references.isEmpty()) { + return Map.of(); + } else { + Map result = new LinkedHashMap<>(); + resolveReferences(result, references, loader); + return result; + } + } + + /** + * Recursively resolve the references. Instead of using the reference name as the key, it uses the full + * coordinates of the artifact version. Re-writes each schema node content to use the full coordinates of + * the artifact version instead of just using the original reference name. + * + * @return the main content rewritten to use the full coordinates of the artifact version and the full + * tree of dependencies, also rewritten to use coordinates instead of the reference name. + */ + public static RewrittenContentHolder recursivelyResolveReferencesWithContext(TypedContent mainContent, + String mainContentType, List references, + Function loader) { + if (references == null || references.isEmpty()) { + return new RewrittenContentHolder(mainContent, Collections.emptyMap()); + } else { + Map resolvedReferences = new LinkedHashMap<>(); + // First we resolve all the references tree, re-writing the nested contents to use the artifact + // version coordinates instead of the reference name. + return resolveReferencesWithContext(mainContent, mainContentType, resolvedReferences, references, + loader); + } + } + + /** + * Recursively resolve the references. Instead of using the reference name as the key, it uses the full + * coordinates of the artifact version. Re-writes each schema node content to use the full coordinates of + * the artifact version instead of just using the original reference name. This allows to dereference json + * schema artifacts where there might be duplicate file names in a single hierarchy. + */ + private static RewrittenContentHolder resolveReferencesWithContext(TypedContent mainContent, + String schemaType, Map partialRecursivelyResolvedReferences, + List references, Function loader) { + Map referencesRewrites = new HashMap<>(); + if (references != null && !references.isEmpty()) { + for (ArtifactReferenceDto reference : references) { + if (reference.getArtifactId() == null || reference.getName() == null + || reference.getVersion() == null) { + throw new IllegalStateException("Invalid reference: " + reference); + } else { + String refName = reference.getName(); + String referenceCoordinates = concatArtifactVersionCoordinatesWithRefName( + reference.getGroupId(), reference.getArtifactId(), reference.getVersion(), + refName); + + JsonPointerExternalReference refPointer = new JsonPointerExternalReference(refName); + JsonPointerExternalReference coordinatePointer = new JsonPointerExternalReference( + referenceCoordinates, refPointer.getComponent()); + + String newRefName = coordinatePointer.toString(); + + if (!partialRecursivelyResolvedReferences.containsKey(newRefName)) { + try { + var nested = loader.apply(reference); + if (nested != null) { + ArtifactTypeUtilProvider typeUtilProvider = ARTIFACT_TYPE_UTIL + .getArtifactTypeProvider(nested.getArtifactType()); + RewrittenContentHolder rewrittenContentHolder = resolveReferencesWithContext( + TypedContent.create(nested.getContent(), nested.getArtifactType()), + nested.getArtifactType(), partialRecursivelyResolvedReferences, + nested.getReferences(), loader); + referencesRewrites.put(refName, referenceCoordinates); + TypedContent rewrittenContent = typeUtilProvider.getContentDereferencer() + .rewriteReferences(rewrittenContentHolder.getRewrittenContent(), + referencesRewrites); + partialRecursivelyResolvedReferences.put(newRefName, rewrittenContent); + } + } catch (Exception ex) { + log.error("Could not resolve reference " + reference + ".", ex); + } + } + } + } + } + ArtifactTypeUtilProvider typeUtilProvider = ARTIFACT_TYPE_UTIL.getArtifactTypeProvider(schemaType); + TypedContent rewrittenContent = typeUtilProvider.getContentDereferencer() + .rewriteReferences(mainContent, referencesRewrites); + return new RewrittenContentHolder(rewrittenContent, partialRecursivelyResolvedReferences); + } + + private static void resolveReferences(Map partialRecursivelyResolvedReferences, + List references, Function loader) { + if (references != null && !references.isEmpty()) { + for (ArtifactReferenceDto reference : references) { + if (reference.getArtifactId() == null || reference.getName() == null + || reference.getVersion() == null) { + throw new IllegalStateException("Invalid reference: " + reference); + } else { + if (!partialRecursivelyResolvedReferences.containsKey(reference.getName())) { + try { + var nested = loader.apply(reference); + if (nested != null) { + resolveReferences(partialRecursivelyResolvedReferences, + nested.getReferences(), loader); + partialRecursivelyResolvedReferences.put(reference.getName(), + TypedContent.create(nested.getContent(), nested.getArtifactType())); + } + } catch (Exception ex) { + log.error("Could not resolve reference " + reference + ".", ex); + } + } + } + } + } + } + + /** + * Canonicalize the given content. + *

+ * WARNING: Fails silently. + */ + private static TypedContent canonicalizeContent(String artifactType, TypedContent content, + Map recursivelyResolvedReferences) { + try { + return ARTIFACT_TYPE_UTIL.getArtifactTypeProvider(artifactType).getContentCanonicalizer() + .canonicalize(content, recursivelyResolvedReferences); + } catch (Exception ex) { + // TODO: We should consider explicitly failing when a content could not be canonicalized. + // throw new RegistryException("Failed to canonicalize content.", ex); + log.debug("Failed to canonicalize content: {}", content.getContent()); + return content; + } + } + + /** + * Canonicalize the given content. + * + * @throws RegistryException in the case of an error. + */ + public static TypedContent canonicalizeContent(String artifactType, ContentWrapperDto data, + Function loader) { + try { + return canonicalizeContent(artifactType, + TypedContent.create(data.getContent(), data.getArtifactType()), + recursivelyResolveReferences(data.getReferences(), loader)); + } catch (Exception ex) { + throw new RegistryException("Failed to canonicalize content.", ex); + } + } + + /** + * @param loader can be null *if and only if* references are empty. + */ + public static String canonicalContentHash(String artifactType, ContentWrapperDto data, + Function loader) { + try { + if (notEmpty(data.getReferences())) { + String serializedReferences = serializeReferences(data.getReferences()); + TypedContent canonicalContent = canonicalizeContent(artifactType, data, loader); + return DigestUtils.sha256Hex(concatContentAndReferences(canonicalContent.getContent().bytes(), + serializedReferences)); + } else { + TypedContent canonicalContent = canonicalizeContent(artifactType, + TypedContent.create(data.getContent(), data.getArtifactType()), Map.of()); + return DigestUtils.sha256Hex(canonicalContent.getContent().bytes()); + } + } catch (IOException ex) { + throw new RegistryException("Failed to compute canonical content hash.", ex); + } + } + + /** + * data.references may be null + */ + public static String contentHash(ContentWrapperDto data) { + try { + if (notEmpty(data.getReferences())) { + String serializedReferences = serializeReferences(data.getReferences()); + return DigestUtils.sha256Hex( + concatContentAndReferences(data.getContent().bytes(), serializedReferences)); + } else { + return data.getContent().getSha256Hash(); + } + } catch (IOException ex) { + throw new RegistryException("Failed to compute content hash.", ex); + } + } + + private static byte[] concatContentAndReferences(byte[] contentBytes, String serializedReferences) + throws IOException { + if (serializedReferences != null && !serializedReferences.isEmpty()) { + var serializedReferencesBytes = ContentHandle.create(serializedReferences).bytes(); + var bytes = ByteBuffer.allocate(contentBytes.length + serializedReferencesBytes.length); + bytes.put(contentBytes); + bytes.put(serializedReferencesBytes); + return bytes.array(); + } else { + throw new IllegalArgumentException("serializedReferences is null or empty"); + } + } + + /** + * Serializes the given collection of labels to a string for artifactStore in the DB. + * + * @param labels + */ + public static String serializeLabels(Map labels) { + try { + if (labels == null) { + return null; + } + if (labels.isEmpty()) { + return null; + } + return MAPPER.writeValueAsString(labels); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Deserialize the labels from their string form to a List<String> form. + * + * @param labelsStr + */ + @SuppressWarnings("unchecked") + public static Map deserializeLabels(String labelsStr) { + try { + if (StringUtil.isEmpty(labelsStr)) { + return null; + } + return MAPPER.readValue(labelsStr, Map.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Serializes the given collection of references to a string for artifactStore in the DB. + * + * @param references + */ + public static String serializeReferences(List references) { + try { + if (references == null || references.isEmpty()) { + return null; + } + return MAPPER.writeValueAsString(references); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Deserialize the references from their string form to a List form. + * + * @param references + */ + public static List deserializeReferences(String references) { + try { + if (StringUtil.isEmpty(references)) { + return Collections.emptyList(); + } + return MAPPER.readValue(references, new TypeReference>() { + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static String normalizeGroupId(String groupId) { + if (groupId == null || "default".equals(groupId)) { + return NULL_GROUP_ID; + } + return groupId; + } + + public static String denormalizeGroupId(String groupId) { + if (NULL_GROUP_ID.equals(groupId)) { + return null; + } + return groupId; + } + + public static boolean notEmpty(Collection collection) { + return collection != null && !collection.isEmpty(); + } + + public static String concatArtifactVersionCoordinatesWithRefName(String groupId, String artifactId, + String version, String referenceName) { + return groupId + ":" + artifactId + ":" + version + ":" + referenceName; + } + + public static class RewrittenContentHolder { + final TypedContent rewrittenContent; + final Map resolvedReferences; + + public RewrittenContentHolder(TypedContent rewrittenContent, + Map resolvedReferences) { + this.rewrittenContent = rewrittenContent; + this.resolvedReferences = resolvedReferences; + } + + public TypedContent getRewrittenContent() { + return rewrittenContent; + } + + public Map getResolvedReferences() { + return resolvedReferences; + } + } +} diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/RegistryStorageContentUtils.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/RegistryStorageContentUtils.java index a7d23c7b72..aec118e087 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/RegistryStorageContentUtils.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/RegistryStorageContentUtils.java @@ -72,7 +72,7 @@ public String getCanonicalContentHash(TypedContent content, String artifactType, Function, Map> referenceResolver) { try { if (notEmpty(references)) { - String referencesSerialized = SqlUtil.serializeReferences(references); + String referencesSerialized = RegistryContentUtils.serializeReferences(references); TypedContent canonicalContent = canonicalizeContent(artifactType, content, referenceResolver.apply(references)); return DigestUtils.sha256Hex(concatContentAndReferences(canonicalContent.getContent().bytes(), @@ -92,7 +92,7 @@ public String getCanonicalContentHash(TypedContent content, String artifactType, public String getContentHash(TypedContent content, List references) { try { if (notEmpty(references)) { - String referencesSerialized = SqlUtil.serializeReferences(references); + String referencesSerialized = RegistryContentUtils.serializeReferences(references); return DigestUtils.sha256Hex( concatContentAndReferences(content.getContent().bytes(), referencesSerialized)); } else { diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/SqlUtil.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/SqlUtil.java deleted file mode 100644 index 81652c9e09..0000000000 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/SqlUtil.java +++ /dev/null @@ -1,95 +0,0 @@ -package io.apicurio.registry.storage.impl.sql; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.apicurio.registry.model.GroupId; -import io.apicurio.registry.storage.dto.ArtifactReferenceDto; -import io.apicurio.registry.utils.StringUtil; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class SqlUtil { - - private static final ObjectMapper mapper = new ObjectMapper(); - - /** - * Serializes the given collection of labels to a string for artifactStore in the DB. - * - * @param labels - */ - public static String serializeLabels(Map labels) { - try { - if (labels == null) { - return null; - } - if (labels.isEmpty()) { - return null; - } - return mapper.writeValueAsString(labels); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - /** - * Deserialize the labels from their string form to a Map form. - * - * @param labelsStr - */ - @SuppressWarnings("unchecked") - public static Map deserializeLabels(String labelsStr) { - try { - if (StringUtil.isEmpty(labelsStr)) { - return null; - } - return mapper.readValue(labelsStr, Map.class); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - /** - * Serializes the given collection of references to a string for artifactStore in the DB. - * - * @param references - */ - public static String serializeReferences(List references) { - try { - if (references == null || references.isEmpty()) { - return null; - } - return mapper.writeValueAsString(references); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - /** - * Deserialize the references from their string form to a List form. - * - * @param references - */ - public static List deserializeReferences(String references) { - try { - if (StringUtil.isEmpty(references)) { - return Collections.emptyList(); - } - return mapper.readValue(references, new TypeReference>() { - }); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - public static String normalizeGroupId(String groupId) { - return new GroupId(groupId).getRawGroupId(); - } - - public static String denormalizeGroupId(String groupId) { - return new GroupId(groupId).getRawGroupIdWithNull(); - } - -} diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactEntityMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactEntityMapper.java index 9adfbe7545..dc0e57537f 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactEntityMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactEntityMapper.java @@ -1,6 +1,6 @@ package io.apicurio.registry.storage.impl.sql.mappers; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; @@ -23,12 +23,12 @@ private ArtifactEntityMapper() { @Override public ArtifactEntity map(ResultSet rs) throws SQLException { ArtifactEntity entity = new ArtifactEntity(); - entity.groupId = SqlUtil.denormalizeGroupId(rs.getString("groupId")); + entity.groupId = RegistryContentUtils.denormalizeGroupId(rs.getString("groupId")); entity.artifactId = rs.getString("artifactId"); entity.artifactType = rs.getString("type"); entity.name = rs.getString("name"); entity.description = rs.getString("description"); - entity.labels = SqlUtil.deserializeLabels(rs.getString("labels")); + entity.labels = RegistryContentUtils.deserializeLabels(rs.getString("labels")); entity.owner = rs.getString("owner"); entity.createdOn = rs.getTimestamp("createdOn").getTime(); entity.modifiedBy = rs.getString("modifiedBy"); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactMetaDataDtoMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactMetaDataDtoMapper.java index 5ae0f59c81..d82eac0682 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactMetaDataDtoMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactMetaDataDtoMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.ArtifactMetaDataDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -23,13 +23,13 @@ private ArtifactMetaDataDtoMapper() { @Override public ArtifactMetaDataDto map(ResultSet rs) throws SQLException { ArtifactMetaDataDto dto = new ArtifactMetaDataDto(); - dto.setGroupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))); + dto.setGroupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))); dto.setArtifactId(rs.getString("artifactId")); dto.setOwner(rs.getString("owner")); dto.setCreatedOn(rs.getTimestamp("createdOn").getTime()); dto.setName(rs.getString("name")); dto.setDescription(rs.getString("description")); - dto.setLabels(SqlUtil.deserializeLabels(rs.getString("labels"))); + dto.setLabels(RegistryContentUtils.deserializeLabels(rs.getString("labels"))); dto.setModifiedBy(rs.getString("modifiedBy")); dto.setModifiedOn(rs.getTimestamp("modifiedOn").getTime()); dto.setArtifactType(rs.getString("type")); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactReferenceDtoMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactReferenceDtoMapper.java index a01ad09d4b..47a531d71a 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactReferenceDtoMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactReferenceDtoMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.ArtifactReferenceDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -23,7 +23,7 @@ private ArtifactReferenceDtoMapper() { @Override public ArtifactReferenceDto map(ResultSet rs) throws SQLException { ArtifactReferenceDto dto = new ArtifactReferenceDto(); - dto.setGroupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))); + dto.setGroupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))); dto.setArtifactId(rs.getString("artifactId")); dto.setVersion(rs.getString("version")); dto.setName(rs.getString("name")); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactRuleEntityMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactRuleEntityMapper.java index 7da74f6adb..fa99452231 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactRuleEntityMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactRuleEntityMapper.java @@ -1,6 +1,6 @@ package io.apicurio.registry.storage.impl.sql.mappers; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; @@ -24,7 +24,7 @@ private ArtifactRuleEntityMapper() { @Override public ArtifactRuleEntity map(ResultSet rs) throws SQLException { ArtifactRuleEntity entity = new ArtifactRuleEntity(); - entity.groupId = SqlUtil.denormalizeGroupId(rs.getString("groupId")); + entity.groupId = RegistryContentUtils.denormalizeGroupId(rs.getString("groupId")); entity.artifactId = rs.getString("artifactId"); entity.type = RuleType.fromValue(rs.getString("type")); entity.configuration = rs.getString("configuration"); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionEntityMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionEntityMapper.java index 7ac5480b0c..06d7cf9006 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionEntityMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionEntityMapper.java @@ -1,6 +1,6 @@ package io.apicurio.registry.storage.impl.sql.mappers; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.types.VersionState; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; @@ -25,7 +25,7 @@ private ArtifactVersionEntityMapper() { public ArtifactVersionEntity map(ResultSet rs) throws SQLException { ArtifactVersionEntity entity = new ArtifactVersionEntity(); entity.globalId = rs.getLong("globalId"); - entity.groupId = SqlUtil.denormalizeGroupId(rs.getString("groupId")); + entity.groupId = RegistryContentUtils.denormalizeGroupId(rs.getString("groupId")); entity.artifactId = rs.getString("artifactId"); entity.version = rs.getString("version"); entity.versionOrder = rs.getInt("versionOrder"); @@ -36,7 +36,7 @@ public ArtifactVersionEntity map(ResultSet rs) throws SQLException { entity.modifiedBy = rs.getString("modifiedBy"); entity.modifiedOn = rs.getTimestamp("modifiedOn").getTime(); entity.state = VersionState.valueOf(rs.getString("state")); - entity.labels = SqlUtil.deserializeLabels(rs.getString("labels")); + entity.labels = RegistryContentUtils.deserializeLabels(rs.getString("labels")); entity.contentId = rs.getLong("contentId"); return entity; } diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionMetaDataDtoMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionMetaDataDtoMapper.java index 5dbdc59a1c..b8bbb037b4 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionMetaDataDtoMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ArtifactVersionMetaDataDtoMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.types.VersionState; @@ -27,7 +27,7 @@ private ArtifactVersionMetaDataDtoMapper() { @Override public ArtifactVersionMetaDataDto map(ResultSet rs) throws SQLException { ArtifactVersionMetaDataDto dto = new ArtifactVersionMetaDataDto(); - dto.setGroupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))); + dto.setGroupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))); dto.setArtifactId(rs.getString("artifactId")); dto.setGlobalId(rs.getLong("globalId")); dto.setContentId(rs.getLong("contentId")); @@ -39,7 +39,7 @@ public ArtifactVersionMetaDataDto map(ResultSet rs) throws SQLException { dto.setVersion(rs.getString("version")); dto.setVersionOrder(rs.getInt("versionOrder")); dto.setArtifactType(rs.getString("type")); - dto.setLabels(SqlUtil.deserializeLabels(rs.getString("labels"))); + dto.setLabels(RegistryContentUtils.deserializeLabels(rs.getString("labels"))); return dto; } diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchEntityMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchEntityMapper.java index e0a6fc1a3b..637291b6b8 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchEntityMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchEntityMapper.java @@ -1,6 +1,6 @@ package io.apicurio.registry.storage.impl.sql.mappers; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.utils.impexp.v3.BranchEntity; @@ -16,7 +16,8 @@ private BranchEntityMapper() { @Override public BranchEntity map(ResultSet rs) throws SQLException { - return BranchEntity.builder().groupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))) + return BranchEntity.builder() + .groupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))) .artifactId(rs.getString("artifactId")).branchId(rs.getString("branchId")) .description(rs.getString("description")).systemDefined(rs.getBoolean("systemDefined")) .owner(rs.getString("owner")).createdOn(rs.getTimestamp("createdOn").getTime()) diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchMetaDataDtoMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchMetaDataDtoMapper.java index 9a8ab6aef3..1c86f0b1a5 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchMetaDataDtoMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/BranchMetaDataDtoMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.BranchMetaDataDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -22,7 +22,8 @@ private BranchMetaDataDtoMapper() { */ @Override public BranchMetaDataDto map(ResultSet rs) throws SQLException { - return BranchMetaDataDto.builder().groupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))) + return BranchMetaDataDto.builder() + .groupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))) .artifactId(rs.getString("artifactId")).branchId(rs.getString("branchId")) .description(rs.getString("description")).systemDefined(rs.getBoolean("systemDefined")) .owner(rs.getString("owner")).createdOn(rs.getTimestamp("createdOn").getTime()) diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ContentMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ContentMapper.java index 3271df93dc..ef6951e0a7 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ContentMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/ContentMapper.java @@ -2,7 +2,7 @@ import io.apicurio.registry.content.ContentHandle; import io.apicurio.registry.storage.dto.ContentWrapperDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -28,7 +28,7 @@ public ContentWrapperDto map(ResultSet rs) throws SQLException { ContentHandle content = ContentHandle.create(contentBytes); contentWrapperDto.setContent(content); contentWrapperDto.setContentType(rs.getString("contentType")); - contentWrapperDto.setReferences(SqlUtil.deserializeReferences(rs.getString("refs"))); + contentWrapperDto.setReferences(RegistryContentUtils.deserializeReferences(rs.getString("refs"))); return contentWrapperDto; } diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupEntityMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupEntityMapper.java index 0ae4baa70f..7a9308dce4 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupEntityMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupEntityMapper.java @@ -1,6 +1,6 @@ package io.apicurio.registry.storage.impl.sql.mappers; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.utils.impexp.v3.GroupEntity; @@ -26,7 +26,7 @@ private GroupEntityMapper() { @Override public GroupEntity map(ResultSet rs) throws SQLException { GroupEntity entity = new GroupEntity(); - entity.groupId = SqlUtil.denormalizeGroupId(rs.getString("groupId")); + entity.groupId = RegistryContentUtils.denormalizeGroupId(rs.getString("groupId")); entity.description = rs.getString("description"); String type = rs.getString("artifactsType"); entity.artifactsType = type; @@ -34,7 +34,7 @@ public GroupEntity map(ResultSet rs) throws SQLException { entity.createdOn = rs.getTimestamp("createdOn").getTime(); entity.modifiedBy = rs.getString("modifiedBy"); entity.modifiedOn = ofNullable(rs.getTimestamp("modifiedOn")).map(Timestamp::getTime).orElse(0L); - entity.labels = SqlUtil.deserializeLabels(rs.getString("labels")); + entity.labels = RegistryContentUtils.deserializeLabels(rs.getString("labels")); return entity; } diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupMetaDataDtoMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupMetaDataDtoMapper.java index 49b7dc47fd..a94a584f04 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupMetaDataDtoMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupMetaDataDtoMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.GroupMetaDataDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -24,7 +24,7 @@ private GroupMetaDataDtoMapper() { @Override public GroupMetaDataDto map(ResultSet rs) throws SQLException { GroupMetaDataDto dto = new GroupMetaDataDto(); - dto.setGroupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))); + dto.setGroupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))); dto.setDescription(rs.getString("description")); String type = rs.getString("artifactsType"); @@ -37,7 +37,7 @@ public GroupMetaDataDto map(ResultSet rs) throws SQLException { Timestamp modifiedOn = rs.getTimestamp("modifiedOn"); dto.setModifiedOn(modifiedOn == null ? 0 : modifiedOn.getTime()); - dto.setLabels(SqlUtil.deserializeLabels(rs.getString("labels"))); + dto.setLabels(RegistryContentUtils.deserializeLabels(rs.getString("labels"))); return dto; } diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupRuleEntityMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupRuleEntityMapper.java index 2c33b923e4..c86dfe57b7 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupRuleEntityMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/GroupRuleEntityMapper.java @@ -1,6 +1,6 @@ package io.apicurio.registry.storage.impl.sql.mappers; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.utils.impexp.v3.GroupRuleEntity; @@ -24,7 +24,7 @@ private GroupRuleEntityMapper() { @Override public GroupRuleEntity map(ResultSet rs) throws SQLException { GroupRuleEntity entity = new GroupRuleEntity(); - entity.groupId = SqlUtil.denormalizeGroupId(rs.getString("groupId")); + entity.groupId = RegistryContentUtils.denormalizeGroupId(rs.getString("groupId")); entity.type = RuleType.fromValue(rs.getString("type")); entity.configuration = rs.getString("configuration"); return entity; diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedArtifactMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedArtifactMapper.java index d81a0e67d6..ef95cf0ce2 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedArtifactMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedArtifactMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.SearchedArtifactDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -23,7 +23,7 @@ private SearchedArtifactMapper() { @Override public SearchedArtifactDto map(ResultSet rs) throws SQLException { SearchedArtifactDto dto = new SearchedArtifactDto(); - dto.setGroupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))); + dto.setGroupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))); dto.setArtifactId(rs.getString("artifactId")); dto.setOwner(rs.getString("owner")); dto.setCreatedOn(rs.getTimestamp("createdOn")); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedBranchMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedBranchMapper.java index d394c91226..f39167e78c 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedBranchMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedBranchMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.SearchedBranchDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -22,7 +22,8 @@ private SearchedBranchMapper() { */ @Override public SearchedBranchDto map(ResultSet rs) throws SQLException { - return SearchedBranchDto.builder().groupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))) + return SearchedBranchDto.builder() + .groupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))) .artifactId(rs.getString("artifactId")).branchId(rs.getString("branchId")) .description(rs.getString("description")).systemDefined(rs.getBoolean("systemDefined")) .owner(rs.getString("owner")).createdOn(rs.getTimestamp("createdOn").getTime()) diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedVersionMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedVersionMapper.java index 48cec20018..f021747bce 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedVersionMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedVersionMapper.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.impl.sql.mappers; import io.apicurio.registry.storage.dto.SearchedVersionDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import io.apicurio.registry.types.VersionState; @@ -24,7 +24,7 @@ private SearchedVersionMapper() { @Override public SearchedVersionDto map(ResultSet rs) throws SQLException { SearchedVersionDto dto = new SearchedVersionDto(); - dto.setGroupId(SqlUtil.denormalizeGroupId(rs.getString("groupId"))); + dto.setGroupId(RegistryContentUtils.denormalizeGroupId(rs.getString("groupId"))); dto.setArtifactId(rs.getString("artifactId")); dto.setVersion(rs.getString("version")); dto.setVersionOrder(rs.getInt("versionOrder")); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/StoredArtifactMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/StoredArtifactMapper.java index e51d3204f7..b9009c97b3 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/StoredArtifactMapper.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/StoredArtifactMapper.java @@ -2,7 +2,7 @@ import io.apicurio.registry.content.ContentHandle; import io.apicurio.registry.storage.dto.StoredArtifactVersionDto; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; import java.sql.ResultSet; @@ -27,6 +27,6 @@ public StoredArtifactVersionDto map(ResultSet rs) throws SQLException { .contentType(rs.getString("contentType")).contentId(rs.getLong("contentId")) .globalId(rs.getLong("globalId")).version(rs.getString("version")) .versionOrder(rs.getInt("versionOrder")) - .references(SqlUtil.deserializeReferences(rs.getString("refs"))).build(); + .references(RegistryContentUtils.deserializeReferences(rs.getString("refs"))).build(); } } diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java b/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java index 981bb5f74a..1d588d4781 100644 --- a/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java +++ b/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java @@ -9,8 +9,8 @@ import io.apicurio.registry.storage.dto.EditableArtifactMetaDataDto; import io.apicurio.registry.storage.error.InvalidArtifactTypeException; import io.apicurio.registry.storage.error.VersionAlreadyExistsException; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.RegistryStorageContentUtils; -import io.apicurio.registry.storage.impl.sql.SqlUtil; import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.types.RegistryException; import io.apicurio.registry.types.VersionState; @@ -160,7 +160,7 @@ public void importContent(ContentEntity entity) { entity.contentId = storage.nextContentId(); } - List references = SqlUtil + List references = RegistryContentUtils .deserializeReferences(entity.serializedReferences); // Recalculate the hash using the current algorithm @@ -169,7 +169,8 @@ public void importContent(ContentEntity entity) { // Try to recalculate the canonical hash - this may fail if the content has references try { - Map resolvedReferences = storage.resolveReferences(references); + Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(references, storage::getContentByReference); entity.artifactType = utils.determineArtifactType(typedContent, null, resolvedReferences); // First we have to recalculate both the canonical hash and the contentHash @@ -299,7 +300,8 @@ private void recalculateCanonicalHash(Long contentId) { TypedContent content = TypedContent.create(wrapperDto.getContent(), wrapperDto.getContentType()); String artifactType = versions.get(0).getArtifactType(); - Map resolvedReferences = storage.resolveReferences(references); + Map resolvedReferences = RegistryContentUtils + .recursivelyResolveReferences(references, storage::getContentByReference); TypedContent canonicalContent = utils.canonicalizeContent(artifactType, content, resolvedReferences); String canonicalHash = DigestUtils.sha256Hex(canonicalContent.getContent().bytes()); diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java b/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java index 396c8df3cc..41650a3db6 100644 --- a/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java +++ b/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java @@ -5,8 +5,8 @@ import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.storage.dto.ArtifactReferenceDto; import io.apicurio.registry.storage.error.VersionAlreadyExistsException; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.storage.impl.sql.RegistryStorageContentUtils; -import io.apicurio.registry.storage.impl.sql.SqlUtil; import io.apicurio.registry.types.RegistryException; import io.apicurio.registry.utils.impexp.Entity; import io.apicurio.registry.utils.impexp.EntityInputStream; @@ -99,7 +99,7 @@ public void importArtifactVersion(ArtifactVersionEntity entity) { @Override public void importContent(ContentEntity entity) { try { - List references = SqlUtil + List references = RegistryContentUtils .deserializeReferences(entity.serializedReferences); if (entity.contentType == null) { @@ -112,7 +112,8 @@ public void importContent(ContentEntity entity) { // We do not need canonicalHash if we have artifactType if (entity.canonicalHash == null && entity.artifactType != null) { TypedContent canonicalContent = utils.canonicalizeContent(entity.artifactType, typedContent, - storage.resolveReferences(references)); + RegistryContentUtils.recursivelyResolveReferences(references, + storage::getContentByReference)); entity.canonicalHash = DigestUtils.sha256Hex(canonicalContent.getContent().bytes()); } diff --git a/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/GroupsResourceTest.java b/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/GroupsResourceTest.java index ef4c832cb6..a29e7cfbde 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/GroupsResourceTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/GroupsResourceTest.java @@ -15,7 +15,7 @@ import io.apicurio.registry.rest.v3.beans.VersionMetaData; import io.apicurio.registry.rules.compatibility.jsonschema.diff.DiffType; import io.apicurio.registry.rules.integrity.IntegrityLevel; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.types.ReferenceType; @@ -1613,7 +1613,8 @@ void testArtifactWithReferences() throws Exception { .extract().as(new TypeRef>() { }); - final String referencesSerialized = SqlUtil.serializeReferences(toReferenceDtos(references)); + final String referencesSerialized = RegistryContentUtils + .serializeReferences(toReferenceDtos(references)); // We calculate the hash using the content itself and the references String contentHash = DigestUtils @@ -1893,8 +1894,8 @@ public void testGetArtifactVersionWithReferences() throws Exception { .queryParam("references", "DEREFERENCE") .get("/registry/v3/groups/{groupId}/artifacts/{artifactId}/versions/branch=latest/content") .then().statusCode(200).body("openapi", equalTo("3.0.2")) - .body("paths.widgets.get.responses.200.content.json.schema.items.$ref", - equalTo("#/components/schemas/Widget")); + .body("paths.widgets.get.responses.200.content.json.schema.items.$ref", equalTo( + "GroupsResourceTest:testGetArtifactVersionWithReferences/ReferencedTypes:1:./referenced-types.json#/components/schemas/Widget")); } } diff --git a/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java b/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java index 4c0e54cde3..94015f749f 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java @@ -25,6 +25,8 @@ import io.apicurio.registry.support.Citizen; import io.apicurio.registry.support.CitizenIdentifier; import io.apicurio.registry.support.City; +import io.apicurio.registry.support.CityQualification; +import io.apicurio.registry.support.IdentifierQualification; import io.apicurio.registry.support.Person; import io.apicurio.registry.support.Qualification; import io.apicurio.registry.types.ArtifactType; @@ -270,10 +272,11 @@ public void testJsonSchemaSerdeMagicByte() throws Exception { @Test public void testJsonSchemaSerdeWithReferences() throws Exception { - InputStream citySchema = getClass().getResourceAsStream("/io/apicurio/registry/util/city.json"); - InputStream citizenSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/citizen.json"); + InputStream citySchema = getClass().getResourceAsStream("/io/apicurio/registry/util/city1.json"); + InputStream citizenSchema = getClass() + .getResourceAsStream("/io/apicurio/registry/util/citizen1.json"); InputStream citizenIdentifier = getClass() - .getResourceAsStream("/io/apicurio/registry/util/citizenIdentifier.json"); + .getResourceAsStream("/io/apicurio/registry/util/citizenIdentifier1.json"); InputStream qualificationSchema = getClass() .getResourceAsStream("/io/apicurio/registry/util/qualification.json"); @@ -415,6 +418,320 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { } } + @Test + public void testJsonSchemaSerdeWithReferencesDeserializerDereferenced() throws Exception { + InputStream citySchema = getClass().getResourceAsStream("/io/apicurio/registry/util/city1.json"); + InputStream citizenSchema = getClass() + .getResourceAsStream("/io/apicurio/registry/util/citizen1.json"); + InputStream citizenIdentifier = getClass() + .getResourceAsStream("/io/apicurio/registry/util/citizenIdentifier1.json"); + InputStream qualificationSchema = getClass() + .getResourceAsStream("/io/apicurio/registry/util/qualification.json"); + + InputStream addressSchema = getClass() + .getResourceAsStream("/io/apicurio/registry/util/sample.address.json"); + + Assertions.assertNotNull(citizenSchema); + Assertions.assertNotNull(citySchema); + Assertions.assertNotNull(citizenIdentifier); + Assertions.assertNotNull(qualificationSchema); + Assertions.assertNotNull(addressSchema); + + String groupId = TestUtils.generateGroupId(); + String cityArtifactId = generateArtifactId(); + String qualificationsId = generateArtifactId(); + String identifierArtifactId = generateArtifactId(); + String addressId = generateArtifactId(); + + createArtifact(groupId, cityArtifactId, ArtifactType.JSON, IoUtil.toString(citySchema), + ContentTypes.APPLICATION_JSON); + + createArtifact(groupId, qualificationsId, ArtifactType.JSON, IoUtil.toString(qualificationSchema), + ContentTypes.APPLICATION_JSON); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference qualificationsReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + qualificationsReference.setVersion("1"); + qualificationsReference.setGroupId(groupId); + qualificationsReference.setArtifactId(qualificationsId); + qualificationsReference.setName("qualification.json"); + + createArtifact(groupId, addressId, ArtifactType.JSON, IoUtil.toString(addressSchema), + ContentTypes.APPLICATION_JSON); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference addressReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + addressReference.setVersion("1"); + addressReference.setGroupId(groupId); + addressReference.setArtifactId(addressId); + addressReference.setName("sample.address.json"); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference cityReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + cityReference.setVersion("1"); + cityReference.setGroupId(groupId); + cityReference.setArtifactId(cityArtifactId); + cityReference.setName("city1.json"); + + createArtifact(groupId, identifierArtifactId, ArtifactType.JSON, IoUtil.toString(citizenIdentifier), + ContentTypes.APPLICATION_JSON); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference identifierReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + identifierReference.setVersion("1"); + identifierReference.setGroupId(groupId); + identifierReference.setArtifactId(identifierArtifactId); + identifierReference.setName("citizenIdentifier1.json"); + + String artifactId = generateArtifactId(); + + createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema), + ContentTypes.APPLICATION_JSON, + List.of(qualificationsReference, cityReference, identifierReference, addressReference)); + + City city = new City("New York", 10001); + CitizenIdentifier identifier = new CitizenIdentifier(123456789); + Citizen citizen = new Citizen("Carles", "Arnal", 23, city, identifier, Collections.emptyList()); + + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient)) { + + Map config = new HashMap<>(); + config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); + config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); + config.put(SerdeConfig.VALIDATION_ENABLED, "true"); + config.put(SerdeConfig.DEREFERENCE_SCHEMA, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); + serializer.configure(config, false); + + deserializer.configure(config, false); + + Headers headers = new RecordHeaders(); + byte[] bytes = serializer.serialize(artifactId, headers, citizen); + + citizen = deserializer.deserialize(artifactId, headers, bytes); + + Assertions.assertEquals("Carles", citizen.getFirstName()); + Assertions.assertEquals("Arnal", citizen.getLastName()); + Assertions.assertEquals(23, citizen.getAge()); + Assertions.assertEquals("New York", citizen.getCity().getName()); + + citizen.setAge(-1); + + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } catch (Exception ignored) { + } + + citizen.setAge(23); + city = new City("Kansas CIty", -31); + citizen.setCity(city); + + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } catch (Exception ignored) { + } + + // invalid identifier present, should fail + identifier = new CitizenIdentifier(-1234356); + citizen.setIdentifier(identifier); + + city = new City("Kansas CIty", 22222); + citizen.setCity(city); + + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } catch (Exception ignored) { + } + + // no identifier present, should pass + citizen.setIdentifier(null); + serializer.serialize(artifactId, new RecordHeaders(), citizen); + + // valid qualification, should pass + citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), + new Qualification(UUID.randomUUID().toString(), 7), + new Qualification(UUID.randomUUID().toString(), 8))); + serializer.serialize(artifactId, new RecordHeaders(), citizen); + + // invalid qualification, should fail + citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), + new Qualification(UUID.randomUUID().toString(), -7), + new Qualification(UUID.randomUUID().toString(), 8))); + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } catch (Exception ignored) { + } + } + } + + @Test + public void testWithReferencesDeserializerDereferencedComplexUsecase() throws Exception { + InputStream citySchema = getClass() + .getResourceAsStream("/io/apicurio/registry/util/types/city/city.json"); + InputStream citizenSchema = getClass().getResourceAsStream("/io/apicurio/registry/util/citizen.json"); + InputStream citizenIdentifier = getClass() + .getResourceAsStream("/io/apicurio/registry/util/types/identifier/citizenIdentifier.json"); + InputStream qualificationSchema = getClass() + .getResourceAsStream("/io/apicurio/registry/util/qualification.json"); + InputStream addressSchema = getClass() + .getResourceAsStream("/io/apicurio/registry/util/sample.address.json"); + + InputStream identifierQuarlification = getClass() + .getResourceAsStream("/io/apicurio/registry/util/types/identifier/qualification.json"); + InputStream cityQualification = getClass() + .getResourceAsStream("/io/apicurio/registry/util/types/city/qualification.json"); + + Assertions.assertNotNull(citizenSchema); + Assertions.assertNotNull(citySchema); + Assertions.assertNotNull(citizenIdentifier); + Assertions.assertNotNull(qualificationSchema); + Assertions.assertNotNull(addressSchema); + Assertions.assertNotNull(identifierQuarlification); + Assertions.assertNotNull(cityQualification); + + String groupId = TestUtils.generateGroupId(); + String cityArtifactId = generateArtifactId(); + String qualificationsId = generateArtifactId(); + String identifierArtifactId = generateArtifactId(); + String addressId = generateArtifactId(); + String identifierQualificationId = generateArtifactId(); + String cityQualificationId = generateArtifactId(); + + // Create the two nested qualification schemas, one for the city, and one for the identifier + createArtifact(groupId, identifierQualificationId, ArtifactType.JSON, + IoUtil.toString(identifierQuarlification), ContentTypes.APPLICATION_JSON); + createArtifact(groupId, cityQualificationId, ArtifactType.JSON, IoUtil.toString(cityQualification), + ContentTypes.APPLICATION_JSON); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference cityQualificationReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + cityQualificationReference.setVersion("1"); + cityQualificationReference.setGroupId(groupId); + cityQualificationReference.setArtifactId(cityQualificationId); + cityQualificationReference.setName("qualification.json"); + + // create the city schema with the reference to its qualification + createArtifactWithReferences(groupId, cityArtifactId, ArtifactType.JSON, IoUtil.toString(citySchema), + ContentTypes.APPLICATION_JSON, List.of(cityQualificationReference)); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference identifierQualificationReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + identifierQualificationReference.setVersion("1"); + identifierQualificationReference.setGroupId(groupId); + identifierQualificationReference.setArtifactId(identifierQualificationId); + identifierQualificationReference.setName("qualification.json"); + + // create the identifier schema with the reference to its qualification + createArtifactWithReferences(groupId, identifierArtifactId, ArtifactType.JSON, + IoUtil.toString(citizenIdentifier), ContentTypes.APPLICATION_JSON, + List.of(identifierQualificationReference)); + + // create the main qualification schema, used for the citizen + createArtifact(groupId, qualificationsId, ArtifactType.JSON, IoUtil.toString(qualificationSchema), + ContentTypes.APPLICATION_JSON); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference qualificationsReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + qualificationsReference.setVersion("1"); + qualificationsReference.setGroupId(groupId); + qualificationsReference.setArtifactId(qualificationsId); + qualificationsReference.setName("qualification.json"); + + createArtifact(groupId, addressId, ArtifactType.JSON, IoUtil.toString(addressSchema), + ContentTypes.APPLICATION_JSON); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference addressReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + addressReference.setVersion("1"); + addressReference.setGroupId(groupId); + addressReference.setArtifactId(addressId); + addressReference.setName("sample.address.json"); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference cityReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + cityReference.setVersion("1"); + cityReference.setGroupId(groupId); + cityReference.setArtifactId(cityArtifactId); + cityReference.setName("types/city/city.json"); + + final io.apicurio.registry.rest.v3.beans.ArtifactReference identifierReference = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); + identifierReference.setVersion("1"); + identifierReference.setGroupId(groupId); + identifierReference.setArtifactId(identifierArtifactId); + identifierReference.setName("types/identifier/citizenIdentifier.json"); + + String artifactId = generateArtifactId(); + + // create the citizen schema, with references to qualifications, city, identifier and address + createArtifactWithReferences(groupId, artifactId, ArtifactType.JSON, IoUtil.toString(citizenSchema), + ContentTypes.APPLICATION_JSON, + List.of(qualificationsReference, cityReference, identifierReference, addressReference)); + + City city = new City("New York", 10001); + CitizenIdentifier identifier = new CitizenIdentifier(123456789); + Citizen citizen = new Citizen("Carles", "Arnal", 23, city, identifier, Collections.emptyList()); + + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient)) { + + Map config = new HashMap<>(); + config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); + config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); + config.put(SerdeConfig.DEREFERENCE_SCHEMA, "true"); + config.put(SerdeConfig.USE_ID, IdOption.globalId.name()); + config.put(SerdeConfig.VALIDATION_ENABLED, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); + serializer.configure(config, false); + + Headers headers = new RecordHeaders(); + byte[] bytes = serializer.serialize(artifactId, headers, citizen); + deserializer.configure(config, false); + + citizen = deserializer.deserialize(artifactId, headers, bytes); + + Assertions.assertEquals("Carles", citizen.getFirstName()); + Assertions.assertEquals("Arnal", citizen.getLastName()); + Assertions.assertEquals(23, citizen.getAge()); + Assertions.assertEquals("New York", citizen.getCity().getName()); + + // invalid qualification, should fail + citizen.setQualifications(List.of(new Qualification(UUID.randomUUID().toString(), 6), + new Qualification(UUID.randomUUID().toString(), -7), + new Qualification(UUID.randomUUID().toString(), 8))); + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } catch (Exception ignored) { + } + + // invalid city qualification, minimum is 10 should fail + city.setQualification(new CityQualification("city_qualification", 9)); + citizen.setCity(city); + citizen.setQualifications(Collections.emptyList()); + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } catch (Exception ignored) { + } + + // valid city qualification, should pass + city.setQualification(new CityQualification("city_qualification", 11)); + citizen.setCity(city); + citizen.setQualifications(Collections.emptyList()); + serializer.serialize(artifactId, new RecordHeaders(), citizen); + + // invalid identifier qualification, minimum is 20, should fail + identifier.setIdentifierQualification(new IdentifierQualification("test_subject", 19)); + citizen.setIdentifier(identifier); + try { + serializer.serialize(artifactId, new RecordHeaders(), citizen); + Assertions.fail(); + } catch (Exception ignored) { + } + + // valid identifier qualification + identifier.setIdentifierQualification(new IdentifierQualification("test_subject", 20)); + citizen.setIdentifier(identifier); + serializer.serialize(artifactId, new RecordHeaders(), citizen); + } + } + @Test public void complexObjectValidation() throws Exception { final String version = "8"; diff --git a/app/src/test/java/io/apicurio/registry/rbac/RegistryClientTest.java b/app/src/test/java/io/apicurio/registry/rbac/RegistryClientTest.java index 2a2e0f9ab3..19831bf090 100644 --- a/app/src/test/java/io/apicurio/registry/rbac/RegistryClientTest.java +++ b/app/src/test/java/io/apicurio/registry/rbac/RegistryClientTest.java @@ -33,7 +33,7 @@ import io.apicurio.registry.rest.client.models.VersionSearchResults; import io.apicurio.registry.rest.client.models.VersionState; import io.apicurio.registry.rest.v2.beans.ArtifactContent; -import io.apicurio.registry.storage.impl.sql.SqlUtil; +import io.apicurio.registry.storage.impl.sql.RegistryContentUtils; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.utils.IoUtil; @@ -928,7 +928,7 @@ public void getContentByHash() throws Exception { createArtifactWithReferences(groupId, secondArtifactId, artifactReferences); - String referencesSerialized = SqlUtil + String referencesSerialized = RegistryContentUtils .serializeReferences(toReferenceDtos(artifactReferences.stream().map(r -> { var ref = new io.apicurio.registry.rest.v3.beans.ArtifactReference(); ref.setArtifactId(r.getArtifactId()); diff --git a/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java b/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java index c70cef13b4..04f1a698e8 100644 --- a/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java +++ b/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java @@ -154,7 +154,6 @@ public class ReadOnlyRegistryStorageTest { entry("resetContentId0", new State(true, RegistryStorage::resetContentId)), entry("resetCommentId0", new State(true, RegistryStorage::resetCommentId)), entry("resetGlobalId0", new State(true, RegistryStorage::resetGlobalId)), - entry("resolveReferences1", new State(false, s -> s.resolveReferences(null))), entry("searchArtifacts5", new State(false, s -> s.searchArtifacts(null, null, null, 0, 0))), entry("searchGroups5", new State(false, s -> s.searchGroups(null, null, null, null, null))), entry("searchVersions5", new State(false, s -> s.searchVersions(null, null, null, 0, 0))), @@ -189,7 +188,8 @@ public class ReadOnlyRegistryStorageTest { entry("triggerSnapshotCreation0", new State(true, RegistryStorage::triggerSnapshotCreation)), entry("createSnapshot1", new State(true, s -> s.createSnapshot(null))), - entry("upgradeData3", new State(true, s -> s.upgradeData(null, false, false)))); + entry("upgradeData3", new State(true, s -> s.upgradeData(null, false, false))), + entry("getContentByReference1", new State(true, s -> s.getContentByReference(null)))); CURRENT_METHODS = Arrays.stream(RegistryStorage.class.getMethods()) .map(m -> m.getName() + m.getParameterCount()).collect(Collectors.toSet()); diff --git a/app/src/test/java/io/apicurio/registry/storage/impl/sql/RegistryContentUtilsTest.java b/app/src/test/java/io/apicurio/registry/storage/impl/sql/RegistryContentUtilsTest.java new file mode 100644 index 0000000000..42722a5ca1 --- /dev/null +++ b/app/src/test/java/io/apicurio/registry/storage/impl/sql/RegistryContentUtilsTest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.storage.impl.sql; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.*; + +/** + * @author eric.wittmann@gmail.com + */ +public class RegistryContentUtilsTest { + + @Test + void testSerializeLabels() { + Map props = new HashMap<>(); + props.put("one", "1"); + props.put("two", "2"); + props.put("three", "3"); + String actual = RegistryContentUtils.serializeLabels(props); + String expected = "{\"one\":\"1\",\"two\":\"2\",\"three\":\"3\"}"; + Assertions.assertEquals(expected, actual); + } + + @Test + void testDeserializeLabels() { + String propsStr = "{\"one\":\"1\",\"two\":\"2\",\"three\":\"3\"}"; + Map actual = RegistryContentUtils.deserializeLabels(propsStr); + Assertions.assertNotNull(actual); + Map expected = new HashMap<>(); + expected.put("one", "1"); + expected.put("two", "2"); + expected.put("three", "3"); + Assertions.assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/app/src/test/java/io/apicurio/registry/storage/impl/sql/SqlUtilTest.java b/app/src/test/java/io/apicurio/registry/storage/impl/sql/SqlUtilTest.java deleted file mode 100644 index cf3572b7d0..0000000000 --- a/app/src/test/java/io/apicurio/registry/storage/impl/sql/SqlUtilTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.apicurio.registry.storage.impl.sql; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.Map; - -class SqlUtilTest { - - /** - * Test method for {@link io.apicurio.registry.storage.impl.sql.SqlUtil#serializeLabels(java.util.Map)}. - */ - @Test - void testSerializeLabels() { - Map labels = new HashMap<>(); - labels.put("one", "1"); - labels.put("two", "2"); - labels.put("three", "3"); - String actual = SqlUtil.serializeLabels(labels); - String expected = "{\"one\":\"1\",\"two\":\"2\",\"three\":\"3\"}"; - Assertions.assertEquals(expected, actual); - } - - /** - * Test method for - * {@link io.apicurio.registry.storage.impl.sql.SqlUtil#deserializeLabels(java.lang.String)}. - */ - @Test - void testDeserializeLabels() { - String labelsStr = "{\"one\":\"1\",\"two\":\"2\",\"three\":\"3\"}"; - Map actual = SqlUtil.deserializeLabels(labelsStr); - Assertions.assertNotNull(actual); - Map expected = new HashMap<>(); - expected.put("one", "1"); - expected.put("two", "2"); - expected.put("three", "3"); - Assertions.assertEquals(expected, actual); - } - -} diff --git a/app/src/test/java/io/apicurio/registry/support/CitizenIdentifier.java b/app/src/test/java/io/apicurio/registry/support/CitizenIdentifier.java index 4f2619a1a8..b65786d0aa 100644 --- a/app/src/test/java/io/apicurio/registry/support/CitizenIdentifier.java +++ b/app/src/test/java/io/apicurio/registry/support/CitizenIdentifier.java @@ -7,6 +7,9 @@ public class CitizenIdentifier { @JsonProperty("identifier") private Integer identifier; + @JsonProperty("qualification") + private IdentifierQualification identifierQualification; + public CitizenIdentifier() { } @@ -14,6 +17,11 @@ public CitizenIdentifier(Integer identifier) { this.identifier = identifier; } + public CitizenIdentifier(Integer identifier, IdentifierQualification identifierQualification) { + this.identifier = identifier; + this.identifierQualification = identifierQualification; + } + public Integer getIdentifier() { return identifier; } @@ -21,4 +29,12 @@ public Integer getIdentifier() { public void setIdentifier(Integer identifier) { this.identifier = identifier; } + + public IdentifierQualification getIdentifierQualification() { + return identifierQualification; + } + + public void setIdentifierQualification(IdentifierQualification identifierQualification) { + this.identifierQualification = identifierQualification; + } } diff --git a/app/src/test/java/io/apicurio/registry/support/City.java b/app/src/test/java/io/apicurio/registry/support/City.java index 30c40ebac5..d5c6167180 100644 --- a/app/src/test/java/io/apicurio/registry/support/City.java +++ b/app/src/test/java/io/apicurio/registry/support/City.java @@ -10,6 +10,9 @@ public class City { @JsonProperty("zipCode") private Integer zipCode; + @JsonProperty("qualification") + private CityQualification qualification; + public City() { } @@ -18,6 +21,12 @@ public City(String name, Integer zipCode) { this.zipCode = zipCode; } + public City(String name, Integer zipCode, CityQualification cityQualification) { + this.name = name; + this.zipCode = zipCode; + this.qualification = cityQualification; + } + public String getName() { return name; } @@ -33,4 +42,12 @@ public Integer getZipCode() { public void setZipCode(Integer zipCode) { this.zipCode = zipCode; } + + public CityQualification getQualification() { + return qualification; + } + + public void setQualification(CityQualification qualification) { + this.qualification = qualification; + } } diff --git a/app/src/test/java/io/apicurio/registry/support/CityQualification.java b/app/src/test/java/io/apicurio/registry/support/CityQualification.java new file mode 100644 index 0000000000..04aaa65799 --- /dev/null +++ b/app/src/test/java/io/apicurio/registry/support/CityQualification.java @@ -0,0 +1,36 @@ +package io.apicurio.registry.support; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class CityQualification { + + @JsonProperty("subject_name") + private String subjectName; + + @JsonProperty("qualification") + private int qualification; + + public CityQualification() { + } + + public CityQualification(String subjectName, int qualification) { + this.subjectName = subjectName; + this.qualification = qualification; + } + + public String getSubjectName() { + return subjectName; + } + + public void setSubjectName(String subjectName) { + this.subjectName = subjectName; + } + + public int getQualification() { + return qualification; + } + + public void setQualification(int qualification) { + this.qualification = qualification; + } +} diff --git a/app/src/test/java/io/apicurio/registry/support/IdentifierQualification.java b/app/src/test/java/io/apicurio/registry/support/IdentifierQualification.java new file mode 100644 index 0000000000..731c4d4ffb --- /dev/null +++ b/app/src/test/java/io/apicurio/registry/support/IdentifierQualification.java @@ -0,0 +1,36 @@ +package io.apicurio.registry.support; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class IdentifierQualification { + + @JsonProperty("subject_name") + private String subjectName; + + @JsonProperty("qualification") + private int qualification; + + public IdentifierQualification() { + } + + public IdentifierQualification(String subjectName, int qualification) { + this.subjectName = subjectName; + this.qualification = qualification; + } + + public String getSubjectName() { + return subjectName; + } + + public void setSubjectName(String subjectName) { + this.subjectName = subjectName; + } + + public int getQualification() { + return qualification; + } + + public void setQualification(int qualification) { + this.qualification = qualification; + } +} diff --git a/app/src/test/resources/io/apicurio/registry/util/citizen.json b/app/src/test/resources/io/apicurio/registry/util/citizen.json index 700fb09a83..daa67f08e4 100644 --- a/app/src/test/resources/io/apicurio/registry/util/citizen.json +++ b/app/src/test/resources/io/apicurio/registry/util/citizen.json @@ -18,10 +18,10 @@ "minimum": 0 }, "city": { - "$ref": "city.json" + "$ref": "types/city/city.json" }, "identifier": { - "$ref": "citizenIdentifier.json" + "$ref": "types/identifier/citizenIdentifier.json" }, "qualifications": { "type": "array", diff --git a/app/src/test/resources/io/apicurio/registry/util/citizen1.json b/app/src/test/resources/io/apicurio/registry/util/citizen1.json new file mode 100644 index 0000000000..abea24149f --- /dev/null +++ b/app/src/test/resources/io/apicurio/registry/util/citizen1.json @@ -0,0 +1,36 @@ +{ + "$id": "https://example.com/citizen1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Citizen", + "type": "object", + "properties": { + "firstName": { + "type": "string", + "description": "The citizen's first name." + }, + "lastName": { + "type": "string", + "description": "The citizen's last name." + }, + "age": { + "description": "Age in years which must be equal to or greater than zero.", + "type": "integer", + "minimum": 0 + }, + "city": { + "$ref": "city1.json" + }, + "identifier": { + "$ref": "citizenIdentifier1.json" + }, + "qualifications": { + "type": "array", + "items": { + "$ref": "qualification.json" + } + } + }, + "required": [ + "city" + ] +} \ No newline at end of file diff --git a/app/src/test/resources/io/apicurio/registry/util/citizenIdentifier.json b/app/src/test/resources/io/apicurio/registry/util/citizenIdentifier1.json similarity index 81% rename from app/src/test/resources/io/apicurio/registry/util/citizenIdentifier.json rename to app/src/test/resources/io/apicurio/registry/util/citizenIdentifier1.json index 3a896e55f0..2b4c20118a 100644 --- a/app/src/test/resources/io/apicurio/registry/util/citizenIdentifier.json +++ b/app/src/test/resources/io/apicurio/registry/util/citizenIdentifier1.json @@ -1,5 +1,5 @@ { - "$id": "https://example.com/citizenIdentifier.json", + "$id": "https://example.com/citizenIdentifier1.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Identifier", "type": "object", diff --git a/app/src/test/resources/io/apicurio/registry/util/city.json b/app/src/test/resources/io/apicurio/registry/util/city1.json similarity index 87% rename from app/src/test/resources/io/apicurio/registry/util/city.json rename to app/src/test/resources/io/apicurio/registry/util/city1.json index a86e3e473c..a02d7ef37e 100644 --- a/app/src/test/resources/io/apicurio/registry/util/city.json +++ b/app/src/test/resources/io/apicurio/registry/util/city1.json @@ -1,5 +1,5 @@ { - "$id": "https://example.com/city.json", + "$id": "https://example.com/city1.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "City", "type": "object", diff --git a/app/src/test/resources/io/apicurio/registry/util/types/city/city.json b/app/src/test/resources/io/apicurio/registry/util/types/city/city.json new file mode 100644 index 0000000000..66a1105c0a --- /dev/null +++ b/app/src/test/resources/io/apicurio/registry/util/types/city/city.json @@ -0,0 +1,20 @@ +{ + "$id": "https://example.com/types/city/city.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "City", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The city's name." + }, + "zipCode": { + "type": "integer", + "description": "The zip code.", + "minimum": 0 + }, + "qualification": { + "$ref": "qualification.json" + } + } +} \ No newline at end of file diff --git a/app/src/test/resources/io/apicurio/registry/util/types/city/qualification.json b/app/src/test/resources/io/apicurio/registry/util/types/city/qualification.json new file mode 100644 index 0000000000..4f19d81a31 --- /dev/null +++ b/app/src/test/resources/io/apicurio/registry/util/types/city/qualification.json @@ -0,0 +1,17 @@ +{ + "$id": "https://example.com/types/city/qualification.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Qualification", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The subject's name" + }, + "qualification": { + "type": "integer", + "description": "The city qualification", + "minimum": 10 + } + } +} \ No newline at end of file diff --git a/app/src/test/resources/io/apicurio/registry/util/types/identifier/citizenIdentifier.json b/app/src/test/resources/io/apicurio/registry/util/types/identifier/citizenIdentifier.json new file mode 100644 index 0000000000..0c4677f84a --- /dev/null +++ b/app/src/test/resources/io/apicurio/registry/util/types/identifier/citizenIdentifier.json @@ -0,0 +1,16 @@ +{ + "$id": "https://example.com/types/identifier/citizenIdentifier.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Identifier", + "type": "object", + "properties": { + "identifier": { + "type": "integer", + "description": "The citizen identifier.", + "minimum": 0 + }, + "qualification": { + "$ref": "qualification.json" + } + } +} \ No newline at end of file diff --git a/app/src/test/resources/io/apicurio/registry/util/types/identifier/qualification.json b/app/src/test/resources/io/apicurio/registry/util/types/identifier/qualification.json new file mode 100644 index 0000000000..931557b9d1 --- /dev/null +++ b/app/src/test/resources/io/apicurio/registry/util/types/identifier/qualification.json @@ -0,0 +1,17 @@ +{ + "$id": "https://example.com/types/identifier/qualification.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Qualification", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The subject's name" + }, + "qualification": { + "type": "integer", + "description": "The identifier qualification", + "minimum": 20 + } + } +} \ No newline at end of file diff --git a/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc b/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc index 504bf25c5b..c9cd394150 100644 --- a/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc +++ b/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc @@ -17,11 +17,6 @@ The following {registry} configuration options are available for each component |`false` |`2.1.4.Final` |Include stack trace in errors responses -|`apicurio.apis.v2.base-href` -|`string` -|`_` -|`2.5.0.Final` -|API base href (URI) |`apicurio.apis.v3.base-href` |`string` |`_` diff --git a/schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java index baf883ac04..800f7f2037 100644 --- a/schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java +++ b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java @@ -2,6 +2,16 @@ public class JsonPointerExternalReference extends ExternalReference { + private static String toFullReference(String resource, String component) { + if (resource == null) { + return component; + } + if (component == null) { + return resource; + } + return resource + component; + } + /** * Constructor. * @@ -11,6 +21,10 @@ public JsonPointerExternalReference(String jsonPointer) { super(jsonPointer, resourceFrom(jsonPointer), componentFrom(jsonPointer)); } + public JsonPointerExternalReference(String resource, String component) { + super(toFullReference(resource, component), resource, component); + } + private static String componentFrom(String jsonPointer) { int idx = jsonPointer.indexOf('#'); if (idx == 0) { diff --git a/schema-util/json/pom.xml b/schema-util/json/pom.xml index 36e4e3de54..aee42c5b04 100644 --- a/schema-util/json/pom.xml +++ b/schema-util/json/pom.xml @@ -48,6 +48,10 @@ com.fasterxml.jackson.datatype jackson-datatype-json-org + + io.vertx + vertx-json-schema + org.projectlombok diff --git a/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java b/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java index 4eba43cf58..162689b5a1 100644 --- a/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java +++ b/schema-util/json/src/main/java/io/apicurio/registry/content/dereference/JsonSchemaDereferencer.java @@ -1,5 +1,6 @@ package io.apicurio.registry.content.dereference; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -11,8 +12,16 @@ import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; import io.apicurio.registry.content.ContentHandle; import io.apicurio.registry.content.TypedContent; -import io.apicurio.registry.types.ContentTypes; +import io.apicurio.registry.content.refs.JsonPointerExternalReference; +import io.vertx.core.json.JsonObject; +import io.vertx.json.schema.Draft; +import io.vertx.json.schema.JsonSchema; +import io.vertx.json.schema.JsonSchemaOptions; +import io.vertx.json.schema.impl.JsonRef; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -21,6 +30,9 @@ public class JsonSchemaDereferencer implements ContentDereferencer { private static final ObjectMapper objectMapper; + private static final Logger log = LoggerFactory.getLogger(JsonSchemaDereferencer.class); + private static final String idKey = "$id"; + private static final String schemaKey = "$schema"; static { objectMapper = new ObjectMapper(); @@ -35,8 +47,53 @@ public class JsonSchemaDereferencer implements ContentDereferencer { @Override public TypedContent dereference(TypedContent content, Map resolvedReferences) { - throw new DereferencingNotSupportedException( - "Content dereferencing is not supported for JSON Schema"); + // Here, when using rewrite, I need the new reference coordinates, using the full artifact coordinates + // and not just the reference name and the old name, to be able to do the re-write. + String id = null; + String schema = null; + + try { + JsonNode contentNode = objectMapper.readTree(content.getContent().content()); + id = contentNode.get(idKey).asText(); + schema = contentNode.get(schemaKey).asText(); + } catch (JsonProcessingException e) { + log.warn("No schema or id provided for schema"); + } + + JsonSchemaOptions jsonSchemaOptions = new JsonSchemaOptions().setBaseUri("http://localhost"); + + if (null != schema) { + jsonSchemaOptions.setDraft(Draft.fromIdentifier(schema)); + } + + Map lookups = new HashMap<>(); + resolveReferences(resolvedReferences, lookups); + JsonObject resolvedSchema = JsonRef.resolve(new JsonObject(content.getContent().content()), lookups); + + if (null != id) { + resolvedSchema.put(idKey, id); + } + + if (schema != null) { + resolvedSchema.put(schemaKey, schema); + } + + return TypedContent.create(ContentHandle.create(resolvedSchema.encodePrettily()), + content.getContentType()); + } + + private void resolveReferences(Map resolvedReferences, + Map lookups) { + resolvedReferences.forEach((referenceName, schema) -> { + JsonPointerExternalReference externalRef = new JsonPointerExternalReference(referenceName); + // Note: when adding to 'lookups', strip away the "component" part of the reference, because the + // vertx library is going to do the lookup ONLY by the resource name, excluding the component + lookups.computeIfAbsent(externalRef.getResource(), (key) -> { + JsonObject resolvedSchema = JsonRef.resolve(new JsonObject(schema.getContent().content()), + lookups); + return JsonSchema.of(resolvedSchema); + }); + }); } /** @@ -49,7 +106,7 @@ public TypedContent rewriteReferences(TypedContent content, Map JsonNode tree = objectMapper.readTree(content.getContent().content()); rewriteIn(tree, resolvedReferenceUrls); String converted = objectMapper.writeValueAsString(objectMapper.treeToValue(tree, Object.class)); - return TypedContent.create(ContentHandle.create(converted), ContentTypes.APPLICATION_JSON); + return TypedContent.create(ContentHandle.create(converted), content.getContentType()); } catch (Exception e) { return content; } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java index 3071f274d8..1fbd972be3 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java @@ -34,4 +34,6 @@ public interface ArtifactTypeUtilProvider { ContentDereferencer getContentDereferencer(); ReferenceFinder getReferenceFinder(); + + boolean supportsReferencesWithContext(); } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProviderFactory.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProviderFactory.java index 542f0517bb..aa0558595d 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProviderFactory.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProviderFactory.java @@ -10,4 +10,6 @@ public interface ArtifactTypeUtilProviderFactory { List getAllArtifactTypeProviders(); + String getArtifactContentType(String type); + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java index cd15f1451e..c220e658bf 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java @@ -76,4 +76,9 @@ public ContentDereferencer getContentDereferencer() { public ReferenceFinder getReferenceFinder() { return new AsyncApiReferenceFinder(); } + + @Override + public boolean supportsReferencesWithContext() { + return true; + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java index a0d1465bbc..d2c43cbb43 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java @@ -7,7 +7,7 @@ import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.AvroContentExtractor; import io.apicurio.registry.content.extract.ContentExtractor; -import io.apicurio.registry.content.refs.JsonSchemaReferenceFinder; +import io.apicurio.registry.content.refs.AvroReferenceFinder; import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.content.util.ContentTypeUtil; import io.apicurio.registry.rules.compatibility.AvroCompatibilityChecker; @@ -91,6 +91,12 @@ public ContentDereferencer getContentDereferencer() { @Override public ReferenceFinder getReferenceFinder() { - return new JsonSchemaReferenceFinder(); + return new AvroReferenceFinder(); } + + @Override + public boolean supportsReferencesWithContext() { + return false; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/DefaultArtifactTypeUtilProviderImpl.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/DefaultArtifactTypeUtilProviderImpl.java index bc82e7f6ae..6c4759db43 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/DefaultArtifactTypeUtilProviderImpl.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/DefaultArtifactTypeUtilProviderImpl.java @@ -1,5 +1,8 @@ package io.apicurio.registry.types.provider; +import io.apicurio.registry.types.ArtifactType; +import io.apicurio.registry.types.ContentTypes; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -33,4 +36,22 @@ public List getAllArtifactTypes() { public List getAllArtifactTypeProviders() { return providers; } + + @Override + public String getArtifactContentType(String type) { + // The content-type will be different for protobuf artifacts, graphql artifacts, and XML artifacts + String contentType = ContentTypes.APPLICATION_JSON; + if (type.equals(ArtifactType.PROTOBUF)) { + contentType = ContentTypes.APPLICATION_PROTOBUF; + } + if (type.equals(ArtifactType.GRAPHQL)) { + contentType = ContentTypes.APPLICATION_GRAPHQL; + } + if (type.equals(ArtifactType.WSDL) || type.equals(ArtifactType.XSD) + || type.equals(ArtifactType.XML)) { + contentType = ContentTypes.APPLICATION_XML; + } + + return contentType; + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java index 342c912540..5f1c723c13 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java @@ -74,4 +74,10 @@ public ContentDereferencer getContentDereferencer() { public ReferenceFinder getReferenceFinder() { return NoOpReferenceFinder.INSTANCE; } + + @Override + public boolean supportsReferencesWithContext() { + return false; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java index 4ff0af0df5..5ca65b62d3 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java @@ -4,8 +4,8 @@ import io.apicurio.registry.content.TypedContent; import io.apicurio.registry.content.canon.ContentCanonicalizer; import io.apicurio.registry.content.canon.JsonContentCanonicalizer; -import io.apicurio.registry.content.dereference.AsyncApiDereferencer; import io.apicurio.registry.content.dereference.ContentDereferencer; +import io.apicurio.registry.content.dereference.JsonSchemaDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.JsonContentExtractor; import io.apicurio.registry.content.refs.JsonSchemaReferenceFinder; @@ -66,11 +66,17 @@ protected ContentExtractor createContentExtractor() { @Override public ContentDereferencer getContentDereferencer() { - return new AsyncApiDereferencer(); + return new JsonSchemaDereferencer(); } @Override public ReferenceFinder getReferenceFinder() { return new JsonSchemaReferenceFinder(); } + + @Override + public boolean supportsReferencesWithContext() { + return true; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java index a6348054a9..d546c7eeee 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java @@ -59,4 +59,10 @@ public ContentDereferencer getContentDereferencer() { public ReferenceFinder getReferenceFinder() { return NoOpReferenceFinder.INSTANCE; } + + @Override + public boolean supportsReferencesWithContext() { + return false; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java index 245121485c..019f03a7eb 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java @@ -4,8 +4,8 @@ import io.apicurio.registry.content.TypedContent; import io.apicurio.registry.content.canon.ContentCanonicalizer; import io.apicurio.registry.content.canon.OpenApiContentCanonicalizer; -import io.apicurio.registry.content.dereference.AsyncApiDereferencer; import io.apicurio.registry.content.dereference.ContentDereferencer; +import io.apicurio.registry.content.dereference.OpenApiDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.OpenApiContentExtractor; import io.apicurio.registry.content.refs.OpenApiReferenceFinder; @@ -69,11 +69,17 @@ protected ContentExtractor createContentExtractor() { @Override public ContentDereferencer getContentDereferencer() { - return new AsyncApiDereferencer(); + return new OpenApiDereferencer(); } @Override public ReferenceFinder getReferenceFinder() { return new OpenApiReferenceFinder(); } + + @Override + public boolean supportsReferencesWithContext() { + return true; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java index 1553a41c5a..04d1d9cf9c 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java @@ -79,4 +79,10 @@ public ContentDereferencer getContentDereferencer() { public ReferenceFinder getReferenceFinder() { return new ProtobufReferenceFinder(); } + + @Override + public boolean supportsReferencesWithContext() { + return false; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java index 55c5e3ff61..10ae62b25e 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java @@ -94,4 +94,10 @@ public ContentDereferencer getContentDereferencer() { public ReferenceFinder getReferenceFinder() { return NoOpReferenceFinder.INSTANCE; } + + @Override + public boolean supportsReferencesWithContext() { + return false; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java index 0a57a9e6c7..b1bcac6504 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java @@ -98,4 +98,10 @@ public ContentDereferencer getContentDereferencer() { public ReferenceFinder getReferenceFinder() { return NoOpReferenceFinder.INSTANCE; } + + @Override + public boolean supportsReferencesWithContext() { + return false; + } + } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java index 53f823f69b..63528a15b1 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java @@ -93,4 +93,10 @@ public ContentDereferencer getContentDereferencer() { public ReferenceFinder getReferenceFinder() { return NoOpReferenceFinder.INSTANCE; } + + @Override + public boolean supportsReferencesWithContext() { + return false; + } + } diff --git a/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java index 98b6017a2f..103dcb853c 100644 --- a/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java +++ b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java @@ -47,7 +47,7 @@ public void configure(Map configs, boolean isKey) { @Override public T deserialize(String topic, Headers headers, byte[] data) { - if (headers != null + if (serdeHeaders != null && headers != null && ((JsonSchemaDeserializer) delegatedDeserializer).getSpecificReturnClass() == null) { String javaType = serdeHeaders.getMessageType(headers); ((JsonSchemaDeserializer) delegatedDeserializer)