From 5eec328d7e864debe8b60e3eaed8feafcd1e0de3 Mon Sep 17 00:00:00 2001 From: Eric Wittmann Date: Fri, 16 Aug 2024 08:41:49 -0400 Subject: [PATCH] Several improvements to the Import/Export process (#5037) * Import now writes ZIP to disk first. All import options now support both v2 and v3 ZIP files. Fails on others. * By default, require a registry to be empty before allowing an import * Add debugging to migration test * code formatting * Make sure to read artifact rules after versions so that importing from v2 works --- .../registry/ImportLifecycleBean.java | 55 ++-- .../registry/rest/v2/AdminResourceImpl.java | 71 +---- .../rest/v2/DownloadsResourceImpl.java | 20 -- .../registry/rest/v2/shared/DataExporter.java | 61 ---- .../registry/rest/v3/AdminResourceImpl.java | 101 +++++-- .../registry/rest/v3/shared/DataExporter.java | 5 +- .../registry/storage/RegistryStorage.java | 7 +- .../ReadOnlyRegistryStorageDecorator.java | 2 +- .../RegistryStorageDecoratorBase.java | 2 +- .../RegistryStorageDecoratorReadOnlyBase.java | 5 + .../storage/impexp/EntityInputStream.java | 14 - .../AbstractReadOnlyRegistryStorage.java | 2 +- .../impl/gitops/GitOpsRegistryStorage.java | 5 + .../kafkasql/KafkaSqlRegistryStorage.java | 2 +- .../impl/sql/AbstractSqlRegistryStorage.java | 11 +- .../storage/impl/sql/CommonSqlStatements.java | 8 + .../storage/impl/sql/SqlStatements.java | 3 + .../storage/importing/DataImporter.java | 2 +- .../ImportExportConfigProperties.java | 33 +++ .../importing/v2/AbstractDataImporter.java | 2 +- .../storage/importing/v2/SqlDataUpgrader.java | 182 ++++++------ .../importing/v3/AbstractDataImporter.java | 16 +- .../storage/importing/v3/SqlDataImporter.java | 68 +---- app/src/main/resources/application.properties | 4 +- .../io/apicurio/registry/MigrationTest.java | 9 + .../noprofile/rest/v3/ImportExportTest.java | 16 +- .../readonly/ReadOnlyRegistryStorageTest.java | 1 + .../io/apicurio/registry/utils/IoUtil.java | 54 ++++ .../src/main/resources/META-INF/openapi.json | 12 + .../ref-registry-all-configs.adoc | 20 ++ .../admin/import_request_builder.go | 14 +- go-sdk/pkg/registryclient-v3/kiota-lock.json | 2 +- .../tests/migration/DataMigrationIT.java | 27 +- .../migration/DoNotPreserveIdsImportIT.java | 20 +- .../GenerateCanonicalHashImportIT.java | 2 +- .../MigrationTestsDataInitializer.java | 7 +- .../registry/utils/export/Export.java | 4 +- .../registry/utils/export/ExportContext.java | 2 +- utils/importexport/pom.xml | 10 + .../registry/utils/impexp/EntityInfo.java | 29 ++ .../utils/impexp/EntityInputStream.java | 11 + .../utils/impexp/EntityInputStreamImpl.java | 16 ++ .../registry/utils/impexp/EntityReader.java | 272 ++++++++++++++++++ .../utils/impexp/{v3 => }/EntityWriter.java | 29 +- .../utils/impexp/{v3 => }/ManifestEntity.java | 6 +- .../utils/impexp/v2/EntityReader.java | 129 --------- .../registry/utils/impexp/v2/EntityType.java | 29 -- .../utils/impexp/v2/ManifestEntity.java | 45 --- .../utils/impexp/v3/EntityReader.java | 134 --------- 49 files changed, 823 insertions(+), 758 deletions(-) delete mode 100644 app/src/main/java/io/apicurio/registry/rest/v2/shared/DataExporter.java delete mode 100644 app/src/main/java/io/apicurio/registry/storage/impexp/EntityInputStream.java create mode 100644 app/src/main/java/io/apicurio/registry/storage/importing/ImportExportConfigProperties.java create mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInfo.java create mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStream.java create mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStreamImpl.java create mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityReader.java rename utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/{v3 => }/EntityWriter.java (84%) rename utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/{v3 => }/ManifestEntity.java (71%) delete mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityReader.java delete mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityType.java delete mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/ManifestEntity.java delete mode 100644 utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/EntityReader.java diff --git a/app/src/main/java/io/apicurio/registry/ImportLifecycleBean.java b/app/src/main/java/io/apicurio/registry/ImportLifecycleBean.java index fccf45f4c8..921c1f86d5 100644 --- a/app/src/main/java/io/apicurio/registry/ImportLifecycleBean.java +++ b/app/src/main/java/io/apicurio/registry/ImportLifecycleBean.java @@ -1,27 +1,22 @@ package io.apicurio.registry; -import io.apicurio.common.apps.config.Info; +import io.apicurio.registry.rest.ConflictException; +import io.apicurio.registry.rest.v3.AdminResourceImpl; import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.storage.StorageEvent; import io.apicurio.registry.storage.StorageEventType; import io.apicurio.registry.storage.error.ReadOnlyStorageException; -import io.apicurio.registry.storage.impexp.EntityInputStream; +import io.apicurio.registry.storage.importing.ImportExportConfigProperties; import io.apicurio.registry.types.Current; -import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.v3.EntityReader; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.ObservesAsync; import jakarta.inject.Inject; -import org.eclipse.microprofile.config.inject.ConfigProperty; import org.slf4j.Logger; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Optional; -import java.util.zip.ZipInputStream; @ApplicationScoped public class ImportLifecycleBean { @@ -33,42 +28,30 @@ public class ImportLifecycleBean { @Current RegistryStorage storage; - @ConfigProperty(name = "apicurio.import.url") - @Info(category = "import", description = "The import URL", availableSince = "2.1.0.Final") - Optional registryImportUrlProp; + @Inject + ImportExportConfigProperties importExportProps; + + @Inject + AdminResourceImpl v3Admin; void onStorageReady(@ObservesAsync StorageEvent ev) { - if (StorageEventType.READY.equals(ev.getType()) && registryImportUrlProp.isPresent()) { + if (StorageEventType.READY.equals(ev.getType()) + && importExportProps.registryImportUrlProp.isPresent()) { log.info("Import URL exists."); - final URL registryImportUrl = registryImportUrlProp.get(); + final URL registryImportUrl = importExportProps.registryImportUrlProp.get(); try (final InputStream registryImportZip = new BufferedInputStream( registryImportUrl.openStream())) { log.info("Importing {} on startup.", registryImportUrl); - final ZipInputStream zip = new ZipInputStream(registryImportZip, StandardCharsets.UTF_8); - final EntityReader reader = new EntityReader(zip); - try (EntityInputStream stream = new EntityInputStream() { - @Override - public Entity nextEntity() { - try { - return reader.readEntity(); - } catch (Exception e) { - log.error("Error reading data from import ZIP file {}.", registryImportUrl, e); - return null; - } - } - - @Override - public void close() throws IOException { - zip.close(); - } - }) { - storage.importData(stream, true, true); - log.info("Registry successfully imported from {}", registryImportUrl); - } catch (ReadOnlyStorageException e) { - log.error("Registry import failed, because the storage is in read-only mode."); - } + v3Admin.importData(null, null, null, registryImportZip); + log.info("Registry successfully imported from {}", registryImportUrl); } catch (IOException ioe) { log.error("Registry import from {} failed", registryImportUrl, ioe); + } catch (ReadOnlyStorageException rose) { + log.error("Registry import failed, because the storage is in read-only mode."); + } catch (ConflictException ce) { + log.info("Import skipped, registry not empty."); + } catch (Exception e) { + log.error("Registry import failed", e); } } } diff --git a/app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java index 62adc56ab9..7a05c21cae 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java @@ -1,10 +1,8 @@ package io.apicurio.registry.rest.v2; -import io.apicurio.common.apps.config.Dynamic; import io.apicurio.common.apps.config.DynamicConfigPropertyDef; import io.apicurio.common.apps.config.DynamicConfigPropertyDto; import io.apicurio.common.apps.config.DynamicConfigPropertyIndex; -import io.apicurio.common.apps.config.Info; import io.apicurio.common.apps.logging.Logged; import io.apicurio.common.apps.logging.audit.Audited; import io.apicurio.registry.auth.Authorized; @@ -16,51 +14,37 @@ import io.apicurio.registry.rest.MissingRequiredParameterException; import io.apicurio.registry.rest.v2.beans.ArtifactTypeInfo; import io.apicurio.registry.rest.v2.beans.ConfigurationProperty; -import io.apicurio.registry.rest.v2.beans.DownloadRef; import io.apicurio.registry.rest.v2.beans.RoleMapping; import io.apicurio.registry.rest.v2.beans.Rule; import io.apicurio.registry.rest.v2.beans.UpdateConfigurationProperty; import io.apicurio.registry.rest.v2.beans.UpdateRole; -import io.apicurio.registry.rest.v2.shared.DataExporter; import io.apicurio.registry.rules.DefaultRuleDeletionException; import io.apicurio.registry.rules.RulesProperties; import io.apicurio.registry.storage.RegistryStorage; -import io.apicurio.registry.storage.dto.DownloadContextDto; -import io.apicurio.registry.storage.dto.DownloadContextType; import io.apicurio.registry.storage.dto.RoleMappingDto; import io.apicurio.registry.storage.dto.RuleConfigurationDto; import io.apicurio.registry.storage.error.ConfigPropertyNotFoundException; import io.apicurio.registry.storage.error.InvalidPropertyValueException; import io.apicurio.registry.storage.error.RuleNotFoundException; -import io.apicurio.registry.storage.impexp.EntityInputStream; +import io.apicurio.registry.storage.importing.ImportExportConfigProperties; import io.apicurio.registry.types.Current; import io.apicurio.registry.types.RoleType; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory; -import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.v2.EntityReader; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.interceptor.Interceptors; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.ws.rs.core.Context; -import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; import org.eclipse.microprofile.config.Config; -import org.eclipse.microprofile.config.inject.ConfigProperty; import org.slf4j.Logger; -import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.zip.ZipInputStream; import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_FOR_BROWSER; import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_NAME; @@ -97,17 +81,12 @@ public class AdminResourceImpl implements AdminResource { Config config; @Inject - DataExporter exporter; + ImportExportConfigProperties importExportProps; - @Context - HttpServletRequest request; - - @Dynamic(label = "Download link expiry", description = "The number of seconds that a generated link to a .zip download file is active before expiring.") - @ConfigProperty(name = "apicurio.download.href.ttl.seconds", defaultValue = "30") - @Info(category = "download", description = "Download link expiry", availableSince = "2.1.2.Final") - Supplier downloadHrefTtl; + @Inject + io.apicurio.registry.rest.v3.AdminResourceImpl v3Admin; - private static final void requireParameter(String parameterName, Object parameterValue) { + private static void requireParameter(String parameterName, Object parameterValue) { if (parameterValue == null) { throw new MissingRequiredParameterException(parameterName); } @@ -124,7 +103,6 @@ public List listArtifactTypes() { ati.setName(t); return ati; }).collect(Collectors.toList()); - } /** @@ -246,26 +224,7 @@ public void deleteGlobalRule(RuleType rule) { @Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin) public void importData(Boolean xRegistryPreserveGlobalId, Boolean xRegistryPreserveContentId, InputStream data) { - final ZipInputStream zip = new ZipInputStream(data, StandardCharsets.UTF_8); - final EntityReader reader = new EntityReader(zip); - EntityInputStream stream = new EntityInputStream() { - @Override - public Entity nextEntity() throws IOException { - try { - return reader.readEntity(); - } catch (Exception e) { - log.error("Error reading data from import ZIP file.", e); - return null; - } - } - - @Override - public void close() throws IOException { - zip.close(); - } - }; - this.storage.upgradeData(stream, isNullOrTrue(xRegistryPreserveGlobalId), - isNullOrTrue(xRegistryPreserveContentId)); + v3Admin.importData(xRegistryPreserveGlobalId, xRegistryPreserveContentId, false, data); } /** @@ -275,20 +234,8 @@ public void close() throws IOException { @Audited(extractParameters = { "0", KEY_FOR_BROWSER }) @Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin) public Response exportData(Boolean forBrowser) { - String acceptHeader = request.getHeader("Accept"); - if (Boolean.TRUE.equals(forBrowser) || MediaType.APPLICATION_JSON.equals(acceptHeader)) { - long expires = System.currentTimeMillis() + (downloadHrefTtl.get() * 1000); - DownloadContextDto downloadCtx = DownloadContextDto.builder().type(DownloadContextType.EXPORT) - .expires(expires).build(); - String downloadId = storage.createDownload(downloadCtx); - String downloadHref = createDownloadHref(downloadId); - DownloadRef downloadRef = new DownloadRef(); - downloadRef.setDownloadId(downloadId); - downloadRef.setHref(downloadHref); - return Response.ok(downloadRef).type(MediaType.APPLICATION_JSON_TYPE).build(); - } else { - return exporter.exportData(); - } + throw new UnsupportedOperationException( + "Exporting data using the Registry Core v2 API is no longer supported. Use the v3 API."); } /** @@ -463,7 +410,7 @@ private ConfigurationProperty defToConfigurationProperty(DynamicConfigPropertyDe /** * Lookup the dynamic configuration property being set. Ensure that it exists (throws a - * {@link NotFoundException} if it does not. + * {@link io.apicurio.registry.storage.error.NotFoundException} if it does not. * * @param propertyName the name of the dynamic property * @return the dynamic config property definition diff --git a/app/src/main/java/io/apicurio/registry/rest/v2/DownloadsResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v2/DownloadsResourceImpl.java index 4e1139a8ea..b3ca55ea91 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v2/DownloadsResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v2/DownloadsResourceImpl.java @@ -6,14 +6,8 @@ import io.apicurio.registry.auth.AuthorizedStyle; import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck; import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck; -import io.apicurio.registry.rest.v2.shared.DataExporter; -import io.apicurio.registry.storage.RegistryStorage; -import io.apicurio.registry.storage.dto.DownloadContextDto; -import io.apicurio.registry.storage.dto.DownloadContextType; import io.apicurio.registry.storage.error.DownloadNotFoundException; -import io.apicurio.registry.types.Current; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; import jakarta.interceptor.Interceptors; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -27,25 +21,11 @@ @Path("/apis/registry/v2/downloads") public class DownloadsResourceImpl { - @Inject - @Current - RegistryStorage storage; - - @Inject - DataExporter exporter; - @Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.None) @GET @Path("{downloadId}") @Produces("*/*") public Response download(@PathParam("downloadId") String downloadId) { - DownloadContextDto downloadContext = storage.consumeDownload(downloadId); - if (downloadContext.getType() == DownloadContextType.EXPORT) { - return exporter.exportData(); - } - - // TODO support other types of downloads (e.g. download content by contentId) - throw new DownloadNotFoundException(); } diff --git a/app/src/main/java/io/apicurio/registry/rest/v2/shared/DataExporter.java b/app/src/main/java/io/apicurio/registry/rest/v2/shared/DataExporter.java deleted file mode 100644 index 705b77290f..0000000000 --- a/app/src/main/java/io/apicurio/registry/rest/v2/shared/DataExporter.java +++ /dev/null @@ -1,61 +0,0 @@ -package io.apicurio.registry.rest.v2.shared; - -import io.apicurio.registry.storage.RegistryStorage; -import io.apicurio.registry.types.Current; -import io.apicurio.registry.utils.impexp.v3.EntityWriter; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.StreamingOutput; -import org.slf4j.Logger; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.ZipOutputStream; - -@ApplicationScoped -public class DataExporter { - - @Inject - Logger log; - - @Inject - @Current - RegistryStorage storage; - - /** - * Exports all registry data. - */ - public Response exportData() { - StreamingOutput stream = os -> { - try { - ZipOutputStream zip = new ZipOutputStream(os, StandardCharsets.UTF_8); - EntityWriter writer = new EntityWriter(zip); - AtomicInteger errorCounter = new AtomicInteger(0); - storage.exportData(entity -> { - try { - writer.writeEntity(entity); - } catch (Exception e) { - // TODO do something interesting with this - e.printStackTrace(); - errorCounter.incrementAndGet(); - } - return null; - }); - - // TODO if the errorCounter > 0, then what? - - zip.flush(); - zip.close(); - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new IOException(e); - } - }; - - return Response.ok(stream).type("application/zip").build(); - } - -} diff --git a/app/src/main/java/io/apicurio/registry/rest/v3/AdminResourceImpl.java b/app/src/main/java/io/apicurio/registry/rest/v3/AdminResourceImpl.java index 4da1f4996d..e6e6f2a6d1 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v3/AdminResourceImpl.java +++ b/app/src/main/java/io/apicurio/registry/rest/v3/AdminResourceImpl.java @@ -13,6 +13,7 @@ import io.apicurio.registry.auth.RoleBasedAccessApiOperation; import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck; import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck; +import io.apicurio.registry.rest.ConflictException; import io.apicurio.registry.rest.MissingRequiredParameterException; import io.apicurio.registry.rest.v3.beans.ArtifactTypeInfo; import io.apicurio.registry.rest.v3.beans.ConfigurationProperty; @@ -36,19 +37,26 @@ import io.apicurio.registry.storage.error.ConfigPropertyNotFoundException; import io.apicurio.registry.storage.error.InvalidPropertyValueException; import io.apicurio.registry.storage.error.RuleNotFoundException; -import io.apicurio.registry.storage.impexp.EntityInputStream; +import io.apicurio.registry.storage.importing.ImportExportConfigProperties; import io.apicurio.registry.types.Current; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory; +import io.apicurio.registry.utils.IoUtil; import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.v3.EntityReader; +import io.apicurio.registry.utils.impexp.EntityInputStream; +import io.apicurio.registry.utils.impexp.EntityInputStreamImpl; +import io.apicurio.registry.utils.impexp.EntityReader; +import io.apicurio.registry.utils.impexp.EntityType; +import io.apicurio.registry.utils.impexp.ManifestEntity; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.interceptor.Interceptors; import jakarta.servlet.http.HttpServletRequest; +import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import org.apache.commons.io.FileUtils; import org.eclipse.microprofile.config.Config; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.slf4j.Logger; @@ -57,6 +65,9 @@ import java.io.InputStream; import java.math.BigInteger; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -103,6 +114,9 @@ public class AdminResourceImpl implements AdminResource { @Inject DataExporter exporter; + @Inject + ImportExportConfigProperties importExportProps; + @Context HttpServletRequest request; @@ -250,33 +264,82 @@ public void deleteGlobalRule(RuleType rule) { } /** - * @see io.apicurio.registry.rest.v3.AdminResource#importData(Boolean, Boolean, java.io.InputStream) + * @see io.apicurio.registry.rest.v3.AdminResource#importData(Boolean, Boolean, Boolean, InputStream) */ @Override @Audited @Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin) public void importData(Boolean xRegistryPreserveGlobalId, Boolean xRegistryPreserveContentId, - InputStream data) { + Boolean requireEmptyRegistry, InputStream data) { + boolean preserveGlobalId = xRegistryPreserveGlobalId == null ? importExportProps.preserveGlobalId + : xRegistryPreserveGlobalId; + boolean preserveContentId = xRegistryPreserveContentId == null ? importExportProps.preserveContentId + : xRegistryPreserveContentId; + boolean requireEmpty = requireEmptyRegistry == null ? importExportProps.requireEmptyRegistry + : requireEmptyRegistry; + + if (requireEmpty && !storage.isEmpty()) { + throw new ConflictException("Registry is not empty."); + } + + // The input should be a ZIP file final ZipInputStream zip = new ZipInputStream(data, StandardCharsets.UTF_8); - final EntityReader reader = new EntityReader(zip); - EntityInputStream stream = new EntityInputStream() { - @Override - public Entity nextEntity() throws IOException { - try { - return reader.readEntity(); - } catch (Exception e) { - log.error("Error reading data from import ZIP file.", e); - return null; + + // Unpack the ZIP file to the local file system (temp) + Path tempDirectory = null; + try { + tempDirectory = Files.createTempDirectory(Paths.get(importExportProps.workDir), + "apicurio-import_"); + IoUtil.unpackToDisk(zip, tempDirectory); + zip.close(); + } catch (IOException e) { + throw new BadRequestException("Error importing data: " + e.getMessage(), e); + } + + try { + // EntityReader reader reads all unpacked entities from the file system + final EntityReader reader = new EntityReader(tempDirectory); + + // Check the manifest for the version of the ZIP. We either need to import + // or import with upgrade depending on the version. + boolean upgrade = false; + try { + Entity entity = reader.readNextEntity(); + if (entity.getEntityType() != EntityType.Manifest) { + throw new BadRequestException("Invalid import file: missing Manifest file"); } + ManifestEntity manifestEntity = (ManifestEntity) entity; + + // Version 2 or 1 requires an upgrade to v3. + if (manifestEntity.exportVersion.startsWith("3")) { + upgrade = false; + } else if (manifestEntity.exportVersion.startsWith("2") + || manifestEntity.exportVersion.startsWith("1")) { + upgrade = true; + } else { + throw new BadRequestException( + "Invalid import file, unknown manifest version: " + manifestEntity.systemVersion); + } + } catch (IOException e) { + throw new BadRequestException("Error importing data: " + e.getMessage(), e); } - @Override - public void close() throws IOException { - zip.close(); + // Create an entity input stream to pass to the storage layer + EntityInputStream stream = new EntityInputStreamImpl(reader); + + // Import or upgrade the data into the storage + if (upgrade) { + this.storage.upgradeData(stream, preserveGlobalId, preserveContentId); + } else { + this.storage.importData(stream, preserveGlobalId, preserveContentId); + } + } finally { + try { + FileUtils.deleteDirectory(tempDirectory.toFile()); + } catch (IOException e) { + // Best effort } - }; - this.storage.importData(stream, isNullOrTrue(xRegistryPreserveGlobalId), - isNullOrTrue(xRegistryPreserveContentId)); + } } /** diff --git a/app/src/main/java/io/apicurio/registry/rest/v3/shared/DataExporter.java b/app/src/main/java/io/apicurio/registry/rest/v3/shared/DataExporter.java index f20277496b..f5cd2d184d 100644 --- a/app/src/main/java/io/apicurio/registry/rest/v3/shared/DataExporter.java +++ b/app/src/main/java/io/apicurio/registry/rest/v3/shared/DataExporter.java @@ -2,7 +2,7 @@ import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.types.Current; -import io.apicurio.registry.utils.impexp.v3.EntityWriter; +import io.apicurio.registry.utils.impexp.EntityWriter; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; @@ -37,8 +37,7 @@ public Response exportData() { try { writer.writeEntity(entity); } catch (Exception e) { - // TODO do something interesting with this - e.printStackTrace(); + log.error("Error writing entity", e); errorCounter.incrementAndGet(); } return null; diff --git a/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java index c68e69df1e..4872e450e8 100644 --- a/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/RegistryStorage.java @@ -40,9 +40,9 @@ import io.apicurio.registry.storage.error.RuleNotFoundException; import io.apicurio.registry.storage.error.VersionAlreadyExistsException; import io.apicurio.registry.storage.error.VersionNotFoundException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.utils.impexp.Entity; +import io.apicurio.registry.utils.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; @@ -97,6 +97,11 @@ public interface RegistryStorage extends DynamicConfigStorage { */ boolean isReadOnly(); + /** + * Returns true if the storage is empty (and ready for data to be imported). + */ + boolean isEmpty(); + /** * Create a new artifact in the storage, with or without an initial/first version. Throws an exception if * the artifact already exists. The first version information can be null, in which case an empty artifact diff --git a/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java b/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java index f94ccf9427..b2a2d9800e 100644 --- a/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java +++ b/app/src/main/java/io/apicurio/registry/storage/decorator/ReadOnlyRegistryStorageDecorator.java @@ -26,8 +26,8 @@ import io.apicurio.registry.storage.error.RegistryStorageException; import io.apicurio.registry.storage.error.RuleAlreadyExistsException; import io.apicurio.registry.storage.error.RuleNotFoundException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.types.RuleType; +import io.apicurio.registry.utils.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; diff --git a/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorBase.java b/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorBase.java index 6679f0a142..5762b4c809 100644 --- a/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorBase.java +++ b/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorBase.java @@ -23,8 +23,8 @@ import io.apicurio.registry.storage.error.RuleAlreadyExistsException; import io.apicurio.registry.storage.error.RuleNotFoundException; import io.apicurio.registry.storage.error.VersionNotFoundException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.types.RuleType; +import io.apicurio.registry.utils.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; diff --git a/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java b/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java index e1d4e98c2e..275598ab41 100644 --- a/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java +++ b/app/src/main/java/io/apicurio/registry/storage/decorator/RegistryStorageDecoratorReadOnlyBase.java @@ -277,6 +277,11 @@ public Map resolveReferences(List re return delegate.resolveReferences(references); } + @Override + public boolean isEmpty() { + return delegate.isEmpty(); + } + @Override public boolean isArtifactExists(String groupId, String artifactId) throws RegistryStorageException { return delegate.isArtifactExists(groupId, artifactId); diff --git a/app/src/main/java/io/apicurio/registry/storage/impexp/EntityInputStream.java b/app/src/main/java/io/apicurio/registry/storage/impexp/EntityInputStream.java deleted file mode 100644 index 9b01c290ec..0000000000 --- a/app/src/main/java/io/apicurio/registry/storage/impexp/EntityInputStream.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.apicurio.registry.storage.impexp; - -import io.apicurio.registry.utils.impexp.Entity; - -import java.io.Closeable; -import java.io.IOException; - -public interface EntityInputStream extends Closeable { - - /** - * Get the next import entity from the stream of entities being imported. - */ - Entity nextEntity() throws IOException; -} diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/gitops/AbstractReadOnlyRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/gitops/AbstractReadOnlyRegistryStorage.java index afbc2a21d4..bdb756de1e 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/gitops/AbstractReadOnlyRegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/gitops/AbstractReadOnlyRegistryStorage.java @@ -19,8 +19,8 @@ import io.apicurio.registry.storage.dto.GroupMetaDataDto; import io.apicurio.registry.storage.dto.RuleConfigurationDto; import io.apicurio.registry.storage.error.RegistryStorageException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.types.RuleType; +import io.apicurio.registry.utils.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java index 910aed31a2..2aa7a87da4 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/gitops/GitOpsRegistryStorage.java @@ -426,6 +426,11 @@ public boolean isArtifactExists(String groupId, String artifactId) { return proxy(storage -> storage.isArtifactExists(groupId, artifactId)); } + @Override + public boolean isEmpty() { + return proxy(storage -> storage.isEmpty()); + } + @Override public boolean isGroupExists(String groupId) { return proxy(storage -> storage.isGroupExists(groupId)); diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java index 9ba8ecc096..397b7d2c43 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/kafkasql/KafkaSqlRegistryStorage.java @@ -21,7 +21,6 @@ import io.apicurio.registry.storage.error.RuleAlreadyExistsException; import io.apicurio.registry.storage.error.RuleNotFoundException; import io.apicurio.registry.storage.error.VersionNotFoundException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.storage.impl.kafkasql.messages.*; import io.apicurio.registry.storage.impl.kafkasql.sql.KafkaSqlSink; import io.apicurio.registry.storage.impl.sql.RegistryStorageContentUtils; @@ -31,6 +30,7 @@ import io.apicurio.registry.storage.importing.v3.SqlDataImporter; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.utils.ConcurrentUtil; +import io.apicurio.registry.utils.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java index f987f298ad..b15f918e4d 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java @@ -63,7 +63,6 @@ import io.apicurio.registry.storage.error.VersionAlreadyExistsException; import io.apicurio.registry.storage.error.VersionAlreadyExistsOnBranchException; import io.apicurio.registry.storage.error.VersionNotFoundException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.storage.impl.sql.jdb.Handle; import io.apicurio.registry.storage.impl.sql.jdb.Query; import io.apicurio.registry.storage.impl.sql.jdb.RowMapper; @@ -102,6 +101,8 @@ import io.apicurio.registry.utils.IoUtil; import io.apicurio.registry.utils.StringUtil; import io.apicurio.registry.utils.impexp.Entity; +import io.apicurio.registry.utils.impexp.EntityInputStream; +import io.apicurio.registry.utils.impexp.ManifestEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; @@ -111,7 +112,6 @@ import io.apicurio.registry.utils.impexp.v3.GlobalRuleEntity; import io.apicurio.registry.utils.impexp.v3.GroupEntity; import io.apicurio.registry.utils.impexp.v3.GroupRuleEntity; -import io.apicurio.registry.utils.impexp.v3.ManifestEntity; import io.quarkus.security.identity.SecurityIdentity; import jakarta.enterprise.event.Event; import jakarta.inject.Inject; @@ -2925,6 +2925,13 @@ public void importComment(CommentEntity entity) { }); } + @Override + public boolean isEmpty() { + return handles.withHandle(handle -> { + return handle.createQuery(sqlStatements.selectAllContentCount()).mapTo(Long.class).one() == 0; + }); + } + private boolean isContentExists(Handle handle, long contentId) { return handle.createQuery(sqlStatements().selectContentExists()).bind(0, contentId) .mapTo(Integer.class).one() > 0; diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/CommonSqlStatements.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/CommonSqlStatements.java index 6858ed3f95..4caa3473a5 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/CommonSqlStatements.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/CommonSqlStatements.java @@ -519,6 +519,14 @@ public String selectAllArtifactCount() { return "SELECT COUNT(a.artifactId) FROM artifacts a "; } + /** + * @see SqlStatements#selectAllContentCount() + */ + @Override + public String selectAllContentCount() { + return "SELECT COUNT(c.contentId) FROM content c "; + } + /** * @see io.apicurio.registry.storage.impl.sql.SqlStatements#selectAllArtifactVersionsCount() */ diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/SqlStatements.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/SqlStatements.java index 2d53254662..2b98ec8ea5 100644 --- a/app/src/main/java/io/apicurio/registry/storage/impl/sql/SqlStatements.java +++ b/app/src/main/java/io/apicurio/registry/storage/impl/sql/SqlStatements.java @@ -501,6 +501,8 @@ public interface SqlStatements { public String selectGlobalIdExists(); + public String selectAllContentCount(); + /* * The next few statements support role mappings */ @@ -616,4 +618,5 @@ public interface SqlStatements { public String createDataSnapshot(); public String restoreFromSnapshot(); + } diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/DataImporter.java b/app/src/main/java/io/apicurio/registry/storage/importing/DataImporter.java index 4865b4539e..860e39b151 100644 --- a/app/src/main/java/io/apicurio/registry/storage/importing/DataImporter.java +++ b/app/src/main/java/io/apicurio/registry/storage/importing/DataImporter.java @@ -1,7 +1,7 @@ package io.apicurio.registry.storage.importing; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.Entity; +import io.apicurio.registry.utils.impexp.EntityInputStream; public interface DataImporter { diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/ImportExportConfigProperties.java b/app/src/main/java/io/apicurio/registry/storage/importing/ImportExportConfigProperties.java new file mode 100644 index 0000000000..e8e1e31fc5 --- /dev/null +++ b/app/src/main/java/io/apicurio/registry/storage/importing/ImportExportConfigProperties.java @@ -0,0 +1,33 @@ +package io.apicurio.registry.storage.importing; + +import io.apicurio.common.apps.config.Info; +import jakarta.inject.Singleton; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import java.net.URL; +import java.util.Optional; + +@Singleton +public class ImportExportConfigProperties { + + @ConfigProperty(name = "apicurio.import.workDir") + @Info(category = "import", description = "Temporary work directory to use when importing data.", availableSince = "3.0.0") + public String workDir; + + @ConfigProperty(name = "apicurio.import.requireEmptyRegistry", defaultValue = "true") + @Info(category = "import", description = "When set to true, importing data will only work when the registry is empty. Defaults to 'true'.", availableSince = "3.0.0") + public boolean requireEmptyRegistry; + + @ConfigProperty(name = "apicurio.import.preserveGlobalId", defaultValue = "true") + @Info(category = "import", description = "When set to true, global IDs from the import file will be used (otherwise new IDs will be generated). Defaults to 'true'.", availableSince = "3.0.0") + public boolean preserveGlobalId; + + @ConfigProperty(name = "apicurio.import.preserveContentId", defaultValue = "true") + @Info(category = "import", description = "When set to true, content IDs from the import file will be used (otherwise new IDs will be generated). Defaults to 'true'.", availableSince = "3.0.0") + public boolean preserveContentId; + + @ConfigProperty(name = "apicurio.import.url") + @Info(category = "import", description = "The import URL", availableSince = "2.1.0.Final") + public Optional registryImportUrlProp; + +} diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/v2/AbstractDataImporter.java b/app/src/main/java/io/apicurio/registry/storage/importing/v2/AbstractDataImporter.java index 83a7d19071..ac940fedb4 100644 --- a/app/src/main/java/io/apicurio/registry/storage/importing/v2/AbstractDataImporter.java +++ b/app/src/main/java/io/apicurio/registry/storage/importing/v2/AbstractDataImporter.java @@ -3,13 +3,13 @@ import io.apicurio.registry.storage.error.RegistryStorageException; import io.apicurio.registry.storage.importing.DataImporter; import io.apicurio.registry.utils.impexp.Entity; +import io.apicurio.registry.utils.impexp.ManifestEntity; import io.apicurio.registry.utils.impexp.v2.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v2.ArtifactVersionEntity; import io.apicurio.registry.utils.impexp.v2.CommentEntity; import io.apicurio.registry.utils.impexp.v2.ContentEntity; import io.apicurio.registry.utils.impexp.v2.GlobalRuleEntity; import io.apicurio.registry.utils.impexp.v2.GroupEntity; -import io.apicurio.registry.utils.impexp.v2.ManifestEntity; import org.slf4j.Logger; public abstract class AbstractDataImporter implements DataImporter { diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java b/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java index 0f3d4e2c51..9a47c4fcee 100644 --- a/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java +++ b/app/src/main/java/io/apicurio/registry/storage/importing/v2/SqlDataUpgrader.java @@ -2,19 +2,20 @@ import io.apicurio.registry.content.ContentHandle; import io.apicurio.registry.content.TypedContent; -import io.apicurio.registry.model.GAV; import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.storage.dto.ArtifactReferenceDto; +import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto; +import io.apicurio.registry.storage.dto.ContentWrapperDto; import io.apicurio.registry.storage.dto.EditableArtifactMetaDataDto; import io.apicurio.registry.storage.error.InvalidArtifactTypeException; import io.apicurio.registry.storage.error.VersionAlreadyExistsException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.storage.impl.sql.RegistryStorageContentUtils; import io.apicurio.registry.storage.impl.sql.SqlUtil; import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.types.RegistryException; import io.apicurio.registry.types.VersionState; import io.apicurio.registry.utils.impexp.Entity; +import io.apicurio.registry.utils.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.v2.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v2.ArtifactVersionEntity; import io.apicurio.registry.utils.impexp.v2.CommentEntity; @@ -26,7 +27,6 @@ import org.slf4j.Logger; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -35,7 +35,15 @@ import java.util.Set; import java.util.stream.Collectors; -import static io.apicurio.registry.types.ArtifactType.*; +import static io.apicurio.registry.types.ArtifactType.ASYNCAPI; +import static io.apicurio.registry.types.ArtifactType.AVRO; +import static io.apicurio.registry.types.ArtifactType.GRAPHQL; +import static io.apicurio.registry.types.ArtifactType.JSON; +import static io.apicurio.registry.types.ArtifactType.OPENAPI; +import static io.apicurio.registry.types.ArtifactType.PROTOBUF; +import static io.apicurio.registry.types.ArtifactType.WSDL; +import static io.apicurio.registry.types.ArtifactType.XML; +import static io.apicurio.registry.types.ArtifactType.XSD; /** * This class takes a stream of Registry v2 entities and imports them into the application using @@ -49,27 +57,15 @@ public class SqlDataUpgrader extends AbstractDataImporter { protected final RegistryStorage storage; protected final boolean preserveGlobalId; - protected final boolean preserveContentId; - // To handle the case where we are trying to import a version before its content has been imported - protected final List waitingForContent = new ArrayList<>(); - - // To handle the case where we are trying to import a comment before its version has been imported - private final List waitingForVersion = new ArrayList<>(); - // ID remapping protected final Map globalIdMapping = new HashMap<>(); protected final Map contentIdMapping = new HashMap<>(); - // Collection of content waiting for required references. A given content cannot be imported unless the - // expected reference is present. - // TODO do a second round to this, since this currently means enforcing the integrity rule. (Maybe try a - // first round and, if there are no artifacts remaining, just import the orphaned content). - protected final Map> waitingForReference = new HashMap<>(); - - // To keep track of which versions have been imported - private final Set gavDone = new HashSet<>(); + // We may need to recalculate the canonical hash for some content after the + // import is complete. + private Set deferredCanonicalHashContentIds = new HashSet<>(); public SqlDataUpgrader(Logger logger, RegistryStorageContentUtils utils, RegistryStorage storage, boolean preserveGlobalId, boolean preserveContentId) { @@ -96,13 +92,6 @@ protected void importArtifactRule(ArtifactRuleEntity entity) { @Override public void importArtifactVersion(ArtifactVersionEntity entity) { try { - // Content needs to be imported before artifact version - if (!contentIdMapping.containsKey(entity.contentId)) { - // Add to the queue waiting for content imported - waitingForContent.add(entity); - return; - } - entity.contentId = contentIdMapping.get(entity.contentId); var oldGlobalId = entity.globalId; @@ -153,33 +142,8 @@ public void importArtifactVersion(ArtifactVersionEntity entity) { } storage.importArtifactVersion(newEntity); - log.debug("Artifact version imported successfully: {}", entity); + log.info("Artifact version imported successfully: {}", entity); globalIdMapping.put(oldGlobalId, entity.globalId); - var gav = new GAV(entity.groupId, entity.artifactId, entity.version); - gavDone.add(gav); - - // Import comments that were waiting for this version - var commentsToImport = waitingForVersion.stream() - .filter(comment -> comment.globalId == oldGlobalId).toList(); - for (CommentEntity commentEntity : commentsToImport) { - importComment(commentEntity); - } - waitingForVersion.removeAll(commentsToImport); - - // Once the artifact version is processed, check if there is some content waiting for this as it's - // reference - // For each content waiting for the version we just inserted, remove it from the list. - waitingForReference.values().forEach(waitingReferences -> waitingReferences.remove(gav)); - - // Finally, once the list of required deps is updated, if it was the last reference needed, import - // the content. - waitingForReference.keySet().stream() - .filter(content -> waitingForReference.get(content).isEmpty()) - .forEach(contentToImport -> { - if (!contentIdMapping.containsKey(contentToImport.contentId)) { - importContent(contentToImport); - } - }); } catch (VersionAlreadyExistsException ex) { if (ex.getGlobalId() != null) { log.warn("Duplicate globalId {} detected, skipping import of artifact version: {}", @@ -195,45 +159,46 @@ public void importArtifactVersion(ArtifactVersionEntity entity) { @Override public void importContent(ContentEntity entity) { try { + // Based on the configuration, a new id is requested or the old one is used. + var oldContentId = entity.contentId; + if (!preserveContentId) { + entity.contentId = storage.nextContentId(); + } + List references = SqlUtil .deserializeReferences(entity.serializedReferences); - Set referencesGavs = references - .stream().map(referenceDto -> new GAV(referenceDto.getGroupId(), - referenceDto.getArtifactId(), referenceDto.getVersion())) - .collect(Collectors.toSet()); - - Set requiredReferences = new HashSet<>(); - - // If there are references and they've not been imported yet, add them to the waiting collection - if (!references.isEmpty() && !gavDone.containsAll(referencesGavs)) { - waitingForReference.put(entity, referencesGavs); - - // For each artifact reference, if it has not been imported yet, add it to the waiting list - // for this content. - referencesGavs.stream() - .filter(artifactReference -> !referencesGavs.contains(artifactReference)) - .forEach(artifactReference -> waitingForReference.get(entity).add(artifactReference)); - - // This content cannot be imported until all the references are imported. - return; - } - + // Recalculate the hash using the current algorithm TypedContent typedContent = TypedContent.create(ContentHandle.create(entity.contentBytes), null); - Map resolvedReferences = storage.resolveReferences(references); - entity.artifactType = utils.determineArtifactType(typedContent, null, resolvedReferences); - - // First we have to recalculate both the canonical hash and the contentHash - TypedContent canonicalContent = utils.canonicalizeContent(entity.artifactType, typedContent, - resolvedReferences); - - entity.canonicalHash = DigestUtils.sha256Hex(canonicalContent.getContent().bytes()); entity.contentHash = utils.getContentHash(typedContent, references); - // Then, based on the configuration, a new id is requested or the old one is used. - var oldContentId = entity.contentId; - if (!preserveContentId) { - entity.contentId = storage.nextContentId(); + // Try to recalculate the canonical hash - this may fail if the content has references + try { + Map resolvedReferences = storage.resolveReferences(references); + entity.artifactType = utils.determineArtifactType(typedContent, null, resolvedReferences); + + // First we have to recalculate both the canonical hash and the contentHash + TypedContent canonicalContent = utils.canonicalizeContent(entity.artifactType, typedContent, + resolvedReferences); + + entity.canonicalHash = DigestUtils.sha256Hex(canonicalContent.getContent().bytes()); + } catch (Exception ex) { + log.debug("Deferring canonical hash calculation: " + ex.getMessage()); + deferredCanonicalHashContentIds.add(entity.contentId); + // Default to AVRO in the case of failure to determine a type (same default that v2 would have + // used). + // Note: the artifactType is not saved to the DB, but is used to determine the content-type. + // So a + // default of AVRO will result in a (sensible) default of application/json for the + // content-type. + // This works for the v2 -> v3 upgrader because only JSON based types will potentially fail + // when + // trying to canonicalize content without fully resolved references. + // + // If this assumption is wrong (e.g. for PROTOBUF) then we'll need an extra step here to + // figure + // out if the core content is JSON or PROTO. + entity.artifactType = AVRO; } // Finally, using the information from the old content, a V3 content entity is created. @@ -247,17 +212,6 @@ public void importContent(ContentEntity entity) { log.debug("Content imported successfully: {}", entity); contentIdMapping.put(oldContentId, entity.contentId); - - // Import artifact versions that were waiting for this content - var artifactsToImport = waitingForContent.stream() - .filter(artifactVersion -> artifactVersion.contentId == oldContentId).toList(); - - for (ArtifactVersionEntity artifactVersionEntity : artifactsToImport) { - artifactVersionEntity.contentId = entity.contentId; - importArtifactVersion(artifactVersionEntity); - } - waitingForContent.removeAll(artifactsToImport); - } catch (Exception ex) { log.warn("Failed to import content {}: {}", entity, ex.getMessage()); } @@ -292,11 +246,6 @@ public void importGroup(GroupEntity entity) { @Override public void importComment(CommentEntity entity) { try { - if (!globalIdMapping.containsKey(entity.globalId)) { - // The version hasn't been imported yet. Need to wait for it. - waitingForVersion.add(entity); - return; - } entity.globalId = globalIdMapping.get(entity.globalId); io.apicurio.registry.utils.impexp.v3.CommentEntity newEntity = io.apicurio.registry.utils.impexp.v3.CommentEntity @@ -332,11 +281,42 @@ public void importData(EntityInputStream entities, Runnable postImportAction) { // Make sure the commentId sequence is set high enough storage.resetCommentId(); + // Recalculate any deferred content IDs + deferredCanonicalHashContentIds.forEach(id -> { + recalculateCanonicalHash(id); + }); + } catch (IOException ex) { throw new RegistryException("Could not read next entity to import", ex); } } + private void recalculateCanonicalHash(Long contentId) { + try { + ContentWrapperDto wrapperDto = storage.getContentById(contentId); + List references = wrapperDto.getReferences(); + + List versions = storage.getArtifactVersionsByContentId(contentId); + if (versions.isEmpty()) { + // Orphaned content - who cares? + return; + } + + TypedContent content = TypedContent.create(wrapperDto.getContent(), wrapperDto.getContentType()); + String artifactType = versions.get(0).getArtifactType(); + Map resolvedReferences = storage.resolveReferences(references); + TypedContent canonicalContent = utils.canonicalizeContent(artifactType, content, + resolvedReferences); + String canonicalHash = DigestUtils.sha256Hex(canonicalContent.getContent().bytes()); + String contentHash = utils.getContentHash(content, references); + + storage.updateContentCanonicalHash(canonicalHash, contentId, contentHash); + } catch (Exception ex) { + // Oh well, we did our best. + log.warn("Failed to recalculate canonical hash for: " + contentId, ex); + } + } + private String determineContentType(String artifactTypeHint, TypedContent content) { if (content.getContentType() != null) { return content.getContentType(); diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/v3/AbstractDataImporter.java b/app/src/main/java/io/apicurio/registry/storage/importing/v3/AbstractDataImporter.java index 7726826677..1faa06b7dd 100644 --- a/app/src/main/java/io/apicurio/registry/storage/importing/v3/AbstractDataImporter.java +++ b/app/src/main/java/io/apicurio/registry/storage/importing/v3/AbstractDataImporter.java @@ -3,7 +3,16 @@ import io.apicurio.registry.storage.error.RegistryStorageException; import io.apicurio.registry.storage.importing.DataImporter; import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.v3.*; +import io.apicurio.registry.utils.impexp.ManifestEntity; +import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; +import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; +import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; +import io.apicurio.registry.utils.impexp.v3.BranchEntity; +import io.apicurio.registry.utils.impexp.v3.CommentEntity; +import io.apicurio.registry.utils.impexp.v3.ContentEntity; +import io.apicurio.registry.utils.impexp.v3.GlobalRuleEntity; +import io.apicurio.registry.utils.impexp.v3.GroupEntity; +import io.apicurio.registry.utils.impexp.v3.GroupRuleEntity; import org.slf4j.Logger; public abstract class AbstractDataImporter implements DataImporter { @@ -38,6 +47,9 @@ public void importEntity(Entity entity) { case Group: importGroup((GroupEntity) entity); break; + case GroupRule: + importGroupRule((GroupRuleEntity) entity); + break; case Comment: importComment((CommentEntity) entity); break; @@ -74,5 +86,7 @@ public void importEntity(Entity entity) { protected abstract void importGroup(GroupEntity entity); + protected abstract void importGroupRule(GroupRuleEntity entity); + protected abstract void importBranch(BranchEntity entity); } diff --git a/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java b/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java index 54723ff7ce..396c8df3cc 100644 --- a/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java +++ b/app/src/main/java/io/apicurio/registry/storage/importing/v3/SqlDataImporter.java @@ -2,15 +2,14 @@ import io.apicurio.registry.content.ContentHandle; import io.apicurio.registry.content.TypedContent; -import io.apicurio.registry.model.GAV; import io.apicurio.registry.storage.RegistryStorage; import io.apicurio.registry.storage.dto.ArtifactReferenceDto; import io.apicurio.registry.storage.error.VersionAlreadyExistsException; -import io.apicurio.registry.storage.impexp.EntityInputStream; import io.apicurio.registry.storage.impl.sql.RegistryStorageContentUtils; import io.apicurio.registry.storage.impl.sql.SqlUtil; import io.apicurio.registry.types.RegistryException; import io.apicurio.registry.utils.impexp.Entity; +import io.apicurio.registry.utils.impexp.EntityInputStream; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; @@ -19,17 +18,14 @@ import io.apicurio.registry.utils.impexp.v3.ContentEntity; import io.apicurio.registry.utils.impexp.v3.GlobalRuleEntity; import io.apicurio.registry.utils.impexp.v3.GroupEntity; +import io.apicurio.registry.utils.impexp.v3.GroupRuleEntity; import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; public class SqlDataImporter extends AbstractDataImporter { @@ -41,20 +37,10 @@ public class SqlDataImporter extends AbstractDataImporter { protected final boolean preserveContentId; - // To handle the case where we are trying to import a version before its content has been imported - protected final List waitingForContent = new ArrayList<>(); - - // To handle the case where we are trying to import a comment before its version has been imported - private final List waitingForVersion = new ArrayList<>(); - // ID remapping protected final Map globalIdMapping = new HashMap<>(); protected final Map contentIdMapping = new HashMap<>(); - // To keep track of which versions have been imported - private final Set gavDone = new HashSet<>(); - private final Map> artifactBranchesWaitingForVersion = new HashMap<>(); - public SqlDataImporter(Logger logger, RegistryStorageContentUtils utils, RegistryStorage storage, boolean preserveGlobalId, boolean preserveContentId) { super(logger); @@ -88,13 +74,6 @@ protected void importArtifact(ArtifactEntity entity) { @Override public void importArtifactVersion(ArtifactVersionEntity entity) { try { - // Content needs to be imported before artifact version - if (!contentIdMapping.containsKey(entity.contentId)) { - // Add to the queue waiting for content imported - waitingForContent.add(entity); - return; - } - entity.contentId = contentIdMapping.get(entity.contentId); var oldGlobalId = entity.globalId; @@ -105,22 +84,6 @@ public void importArtifactVersion(ArtifactVersionEntity entity) { storage.importArtifactVersion(entity); log.debug("Artifact version imported successfully: {}", entity); globalIdMapping.put(oldGlobalId, entity.globalId); - var gav = new GAV(entity.groupId, entity.artifactId, entity.version); - gavDone.add(gav); - - // Import comments that were waiting for this version - var commentsToImport = waitingForVersion.stream() - .filter(comment -> comment.globalId == oldGlobalId).collect(Collectors.toList()); - for (CommentEntity commentEntity : commentsToImport) { - importComment(commentEntity); - } - waitingForVersion.removeAll(commentsToImport); - - // Import branches waiting for version - artifactBranchesWaitingForVersion.computeIfAbsent(gav, _ignored -> List.of()) - .forEach(this::importEntity); - artifactBranchesWaitingForVersion.remove(gav); - } catch (VersionAlreadyExistsException ex) { if (ex.getGlobalId() != null) { log.warn("Duplicate globalId {} detected, skipping import of artifact version: {}", @@ -162,18 +125,6 @@ public void importContent(ContentEntity entity) { log.debug("Content imported successfully: {}", entity); contentIdMapping.put(oldContentId, entity.contentId); - - // Import artifact versions that were waiting for this content - var artifactsToImport = waitingForContent.stream() - .filter(artifactVersion -> artifactVersion.contentId == oldContentId) - .collect(Collectors.toList()); - - for (ArtifactVersionEntity artifactVersionEntity : artifactsToImport) { - artifactVersionEntity.contentId = entity.contentId; - importArtifactVersion(artifactVersionEntity); - } - waitingForContent.removeAll(artifactsToImport); - } catch (Exception ex) { log.warn("Failed to import content {}: {}", entity, ex.getMessage()); } @@ -199,14 +150,19 @@ public void importGroup(GroupEntity entity) { } } + @Override + public void importGroupRule(GroupRuleEntity entity) { + try { + storage.importGroupRule(entity); + log.debug("Group rule imported successfully: {}", entity); + } catch (Exception ex) { + log.warn("Failed to import group rule {}: {}", entity, ex.getMessage()); + } + } + @Override public void importComment(CommentEntity entity) { try { - if (!globalIdMapping.containsKey(entity.globalId)) { - // The version hasn't been imported yet. Need to wait for it. - waitingForVersion.add(entity); - return; - } entity.globalId = globalIdMapping.get(entity.globalId); storage.importComment(entity); diff --git a/app/src/main/resources/application.properties b/app/src/main/resources/application.properties index 0f07eb3ae9..b2b29d476f 100644 --- a/app/src/main/resources/application.properties +++ b/app/src/main/resources/application.properties @@ -159,8 +159,8 @@ apicurio.apis.v2.date-format-timezone=UTC # Storage apicurio.storage.kind=sql - -apicurio.storage.snapshot.location=/tmp/ +apicurio.storage.snapshot.location=${java.io.tmpdir} +apicurio.import.workDir=${java.io.tmpdir} ## SQL Storage apicurio.storage.sql.kind=h2 diff --git a/app/src/test/java/io/apicurio/registry/MigrationTest.java b/app/src/test/java/io/apicurio/registry/MigrationTest.java index e2c663b3fd..08b6e3adf9 100644 --- a/app/src/test/java/io/apicurio/registry/MigrationTest.java +++ b/app/src/test/java/io/apicurio/registry/MigrationTest.java @@ -1,6 +1,9 @@ package io.apicurio.registry; +import io.apicurio.registry.storage.RegistryStorage; +import io.apicurio.registry.types.Current; import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import org.junit.jupiter.api.Test; import java.io.InputStream; @@ -8,8 +11,13 @@ @QuarkusTest public class MigrationTest extends AbstractResourceTestBase { + @Inject + @Current + RegistryStorage storage; + @Test public void migrateData() throws Exception { + storage.deleteAllUserData(); InputStream originalData = getClass().getResource("rest/v3/destination_original_data.zip") .openStream(); @@ -25,6 +33,7 @@ public void migrateData() throws Exception { config.headers.add("Content-Type", "application/zip"); config.headers.add("X-Registry-Preserve-GlobalId", "false"); config.headers.add("X-Registry-Preserve-ContentId", "false"); + config.queryParameters.requireEmptyRegistry = false; }); } } diff --git a/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/ImportExportTest.java b/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/ImportExportTest.java index 55fd7c0a9a..aaf9e2be22 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/ImportExportTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/ImportExportTest.java @@ -80,6 +80,12 @@ public void testExportImport() throws Exception { createGroup.getLabels().setAdditionalData(Map.of("isPrimary", "false")); clientV3.groups().post(createGroup); + // Configure a group rule + CreateRule createRule = new CreateRule(); + createRule.setRuleType(RuleType.INTEGRITY); + createRule.setConfig(IntegrityLevel.ALL_REFS_MAPPED.name()); + clientV3.groups().byGroupId("SecondaryTestGroup").rules().post(createRule); + // Add an empty artifact CreateArtifact createArtifact = new CreateArtifact(); createArtifact.setArtifactId("EmptyArtifact"); @@ -147,7 +153,7 @@ public void testExportImport() throws Exception { .post(createBranch); // Configure some global rules - CreateRule createRule = new CreateRule(); + createRule = new CreateRule(); createRule.setRuleType(RuleType.VALIDITY); createRule.setConfig(ValidityLevel.FULL.name()); clientV3.admin().rules().post(createRule); @@ -217,6 +223,14 @@ public void testExportImport() throws Exception { // TODO: check group labels (not returned by group search) + // Assert group rules + List groupRules = clientV3.groups().byGroupId("SecondaryTestGroup").rules().get(); + Assertions.assertEquals(1, groupRules.size()); + Assertions.assertEquals(List.of(RuleType.INTEGRITY), groupRules); + Rule groupRule = clientV3.groups().byGroupId("SecondaryTestGroup").rules() + .byRuleType(RuleType.INTEGRITY.name()).get(); + Assertions.assertEquals(IntegrityLevel.ALL_REFS_MAPPED.name(), groupRule.getConfig()); + // Assert empty artifact ArtifactMetaData amd = clientV3.groups().byGroupId(groupId).artifacts().byArtifactId("EmptyArtifact") .get(); diff --git a/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java b/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java index cf97134681..c70cef13b4 100644 --- a/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java +++ b/app/src/test/java/io/apicurio/registry/storage/impl/readonly/ReadOnlyRegistryStorageTest.java @@ -134,6 +134,7 @@ public class ReadOnlyRegistryStorageTest { entry("importGroup1", new State(true, s -> s.importGroup(null))), entry("initialize0", new State(false, RegistryStorage::initialize)), entry("isAlive0", new State(false, RegistryStorage::isAlive)), + entry("isEmpty0", new State(false, RegistryStorage::isEmpty)), entry("isArtifactExists2", new State(false, s -> s.isArtifactExists(null, null))), entry("isArtifactRuleExists3", new State(false, s -> s.isArtifactRuleExists(null, null, null))), diff --git a/common/src/main/java/io/apicurio/registry/utils/IoUtil.java b/common/src/main/java/io/apicurio/registry/utils/IoUtil.java index 83d4695ea4..978deaeda0 100644 --- a/common/src/main/java/io/apicurio/registry/utils/IoUtil.java +++ b/common/src/main/java/io/apicurio/registry/utils/IoUtil.java @@ -2,14 +2,54 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; public class IoUtil { + /** + * Writes the given zip file to an output directory (assumed to either not exist or be empty). + * + * @param zip + * @param outputDirectory + */ + public static void unpackToDisk(ZipInputStream zip, Path outputDirectory) throws IOException { + ZipEntry entry = zip.getNextEntry(); + while (entry != null) { + Path entryPath = zipPath(entry, outputDirectory); + if (entry.isDirectory()) { + Files.createDirectories(entryPath); + } else { + Path parentPath = entryPath.getParent(); + if (parentPath != null && Files.notExists(parentPath)) { + Files.createDirectories(parentPath); + } + Files.copy(zip, entryPath, StandardCopyOption.REPLACE_EXISTING); + } + zip.closeEntry(); + entry = zip.getNextEntry(); + } + } + + private static Path zipPath(ZipEntry entry, Path targetDir) throws IOException { + Path targetDirResolved = targetDir.resolve(entry.getName()); + Path normalizedPath = targetDirResolved.normalize(); + if (!normalizedPath.startsWith(targetDir)) { + throw new IOException("Invalid ZIP entry: " + entry.getName()); + } + return normalizedPath; + } + private static ByteArrayOutputStream toBaos(InputStream stream) throws IOException { ByteArrayOutputStream result = new ByteArrayOutputStream(); copy(stream, result); @@ -80,6 +120,20 @@ public static byte[] toBytes(InputStream stream, boolean closeStream) { } } + /** + * Get byte array from a file. + * + * @param file the file + * @return file as a byte array + */ + public static byte[] toBytes(File file) { + try (FileInputStream stream = new FileInputStream(file)) { + return toBaos(stream).toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + /** * Get string from stream. Stream is closed at the end. * diff --git a/common/src/main/resources/META-INF/openapi.json b/common/src/main/resources/META-INF/openapi.json index 672f465073..280cb88df9 100644 --- a/common/src/main/resources/META-INF/openapi.json +++ b/common/src/main/resources/META-INF/openapi.json @@ -565,12 +565,24 @@ }, "in": "header", "required": false + }, + { + "name": "requireEmptyRegistry", + "description": "Query parameter indicating whether the registry must be empty before allowing\ndata to be imported. Defaults to `true` if omitted.", + "schema": { + "type": "boolean" + }, + "in": "query", + "required": false } ], "responses": { "201": { "description": "Indicates that the import was successful." }, + "409": { + "$ref": "#/components/responses/Conflict" + }, "500": { "$ref": "#/components/responses/ServerError" } diff --git a/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc b/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc index 5bbb7f081a..a81397d6ba 100644 --- a/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc +++ b/docs/modules/ROOT/partials/getting-started/ref-registry-all-configs.adoc @@ -379,11 +379,31 @@ The following {registry} configuration options are available for each component |Default |Available from |Description +|`apicurio.import.preserveContentId` +|`boolean` +|`true` +|`3.0.0` +|When set to true, content IDs from the import file will be used (otherwise new IDs will be generated). Defaults to 'true'. +|`apicurio.import.preserveGlobalId` +|`boolean` +|`true` +|`3.0.0` +|When set to true, global IDs from the import file will be used (otherwise new IDs will be generated). Defaults to 'true'. +|`apicurio.import.requireEmptyRegistry` +|`boolean` +|`true` +|`3.0.0` +|When set to true, importing data will only work when the registry is empty. Defaults to 'true'. |`apicurio.import.url` |`optional` | |`2.1.0.Final` |The import URL +|`apicurio.import.workDir` +|`string` +| +|`3.0.0` +|Temporary work directory to use when importing data. |=== == limits diff --git a/go-sdk/pkg/registryclient-v3/admin/import_request_builder.go b/go-sdk/pkg/registryclient-v3/admin/import_request_builder.go index 05348821a5..34a2cfe1c9 100644 --- a/go-sdk/pkg/registryclient-v3/admin/import_request_builder.go +++ b/go-sdk/pkg/registryclient-v3/admin/import_request_builder.go @@ -11,18 +11,26 @@ type ImportRequestBuilder struct { i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.BaseRequestBuilder } +// ImportRequestBuilderPostQueryParameters imports registry data that was previously exported using the `/admin/export` operation. +type ImportRequestBuilderPostQueryParameters struct { + // Query parameter indicating whether the registry must be empty before allowingdata to be imported. Defaults to `true` if omitted. + RequireEmptyRegistry *bool `uriparametername:"requireEmptyRegistry"` +} + // ImportRequestBuilderPostRequestConfiguration configuration for the request such as headers, query parameters, and middleware options. type ImportRequestBuilderPostRequestConfiguration struct { // Request headers Headers *i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestHeaders // Request options Options []i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestOption + // Request query parameters + QueryParameters *ImportRequestBuilderPostQueryParameters } // NewImportRequestBuilderInternal instantiates a new ImportRequestBuilder and sets the default values. func NewImportRequestBuilderInternal(pathParameters map[string]string, requestAdapter i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestAdapter) *ImportRequestBuilder { m := &ImportRequestBuilder{ - BaseRequestBuilder: *i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.NewBaseRequestBuilder(requestAdapter, "{+baseurl}/admin/import", pathParameters), + BaseRequestBuilder: *i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.NewBaseRequestBuilder(requestAdapter, "{+baseurl}/admin/import{?requireEmptyRegistry*}", pathParameters), } return m } @@ -41,6 +49,7 @@ func (m *ImportRequestBuilder) Post(ctx context.Context, body []byte, requestCon return err } errorMapping := i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.ErrorMappings{ + "409": i00eb2e63d156923d00d8e86fe16b5d74daf30e363c9f185a8165cb42aa2f2c71.CreateErrorFromDiscriminatorValue, "500": i00eb2e63d156923d00d8e86fe16b5d74daf30e363c9f185a8165cb42aa2f2c71.CreateErrorFromDiscriminatorValue, } err = m.BaseRequestBuilder.RequestAdapter.SendNoContent(ctx, requestInfo, errorMapping) @@ -54,6 +63,9 @@ func (m *ImportRequestBuilder) Post(ctx context.Context, body []byte, requestCon func (m *ImportRequestBuilder) ToPostRequestInformation(ctx context.Context, body []byte, requestConfiguration *ImportRequestBuilderPostRequestConfiguration) (*i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.RequestInformation, error) { requestInfo := i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.NewRequestInformationWithMethodAndUrlTemplateAndPathParameters(i2ae4187f7daee263371cb1c977df639813ab50ffa529013b7437480d1ec0158f.POST, m.BaseRequestBuilder.UrlTemplate, m.BaseRequestBuilder.PathParameters) if requestConfiguration != nil { + if requestConfiguration.QueryParameters != nil { + requestInfo.AddQueryParameters(*(requestConfiguration.QueryParameters)) + } requestInfo.Headers.AddAll(requestConfiguration.Headers) requestInfo.AddRequestOptions(requestConfiguration.Options) } diff --git a/go-sdk/pkg/registryclient-v3/kiota-lock.json b/go-sdk/pkg/registryclient-v3/kiota-lock.json index 82861a80d2..82b52ce155 100644 --- a/go-sdk/pkg/registryclient-v3/kiota-lock.json +++ b/go-sdk/pkg/registryclient-v3/kiota-lock.json @@ -1,5 +1,5 @@ { - "descriptionHash": "B46056EBAC7BEBE57948B9F0255841628F4C6DC6F79CC57C6A2C964E2E2A7C36A2D8F58110E85785A37D0C304C9023C70B1AB1322AC2267D304EAA3C503DE5C7", + "descriptionHash": "7B7042289B28CB539FFBCB39E2D202C413EC9BC04B2FF368E1C7301E7B8A5D4BE084F17F1803ED552CA1AF09991A30CCCC6B2ED7D9E4437FCA093A2BB33C5BB5", "descriptionLocation": "../../v3.json", "lockFileVersion": "1.0.0", "kiotaVersion": "1.10.1", diff --git a/integration-tests/src/test/java/io/apicurio/tests/migration/DataMigrationIT.java b/integration-tests/src/test/java/io/apicurio/tests/migration/DataMigrationIT.java index 6c90462084..9b2021ccf2 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/migration/DataMigrationIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/migration/DataMigrationIT.java @@ -3,6 +3,7 @@ import io.apicurio.registry.client.auth.VertXAuthFactory; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.rest.client.models.ArtifactReference; +import io.apicurio.registry.rest.client.models.Error; import io.apicurio.registry.types.RuleType; import io.apicurio.registry.utils.tests.TestUtils; import io.apicurio.tests.ApicurioRegistryBaseIT; @@ -56,22 +57,26 @@ public void migrate() throws Exception { given().when().contentType("application/zip").body(migrateDataToImport) .post("/apis/registry/v2/admin/import").then().statusCode(204).body(anything()); - retry(() -> { - for (long gid : migrateGlobalIds) { - dest.ids().globalIds().byGlobalId(gid).get(); - if (migrateReferencesMap.containsKey(gid)) { - List srcReferences = migrateReferencesMap - .get(gid); - List destReferences = dest.ids().globalIds().byGlobalId(gid) - .references().get(); - assertTrue(matchesReferencesV2V3(srcReferences, destReferences)); - } + for (long gid : migrateGlobalIds) { + dest.ids().globalIds().byGlobalId(gid).get(); + if (migrateReferencesMap.containsKey(gid)) { + List srcReferences = migrateReferencesMap + .get(gid); + List destReferences = dest.ids().globalIds().byGlobalId(gid).references() + .get(); + assertTrue(matchesReferencesV2V3(srcReferences, destReferences)); } + } + try { assertEquals("SYNTAX_ONLY", dest.groups().byGroupId("migrateTest").artifacts() .byArtifactId("avro-0").rules().byRuleType(RuleType.VALIDITY.name()).get().getConfig()); assertEquals("BACKWARD", dest.admin().rules().byRuleType(RuleType.COMPATIBILITY.name()).get().getConfig()); - }); + } catch (Error e) { + log.error("REST Client error: " + e.getMessageEscaped()); + log.error(" : " + e.getDetail()); + throw e; + } } public static class MigrateTestInitializer extends AbstractTestDataInitializer { diff --git a/integration-tests/src/test/java/io/apicurio/tests/migration/DoNotPreserveIdsImportIT.java b/integration-tests/src/test/java/io/apicurio/tests/migration/DoNotPreserveIdsImportIT.java index 3e45b972e5..822fbc18b4 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/migration/DoNotPreserveIdsImportIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/migration/DoNotPreserveIdsImportIT.java @@ -98,17 +98,15 @@ public void testDoNotPreserveIdsImport() throws Exception { adapter.sendPrimitive(importReq, new HashMap<>(), Void.class); // Check that the import was successful - retry(() -> { - for (var entry : doNotPreserveIdsImportArtifacts.entrySet()) { - String groupId = entry.getKey().split(":")[0]; - String artifactId = entry.getKey().split(":")[1]; - String content = entry.getValue(); - var registryContent = dest.groups().byGroupId(groupId).artifacts().byArtifactId(artifactId) - .versions().byVersionExpression("branch=latest").content().get(); - assertNotNull(registryContent); - assertEquals(content, IoUtil.toString(registryContent)); - } - }); + for (var entry : doNotPreserveIdsImportArtifacts.entrySet()) { + String groupId = entry.getKey().split(":")[0]; + String artifactId = entry.getKey().split(":")[1]; + String content = entry.getValue(); + var registryContent = dest.groups().byGroupId(groupId).artifacts().byArtifactId(artifactId) + .versions().byVersionExpression("branch=latest").content().get(); + assertNotNull(registryContent); + assertEquals(content, IoUtil.toString(registryContent)); + } } public static class DoNotPreserveIdsInitializer extends AbstractTestDataInitializer { diff --git a/integration-tests/src/test/java/io/apicurio/tests/migration/GenerateCanonicalHashImportIT.java b/integration-tests/src/test/java/io/apicurio/tests/migration/GenerateCanonicalHashImportIT.java index a96e4af1b1..76a2356804 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/migration/GenerateCanonicalHashImportIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/migration/GenerateCanonicalHashImportIT.java @@ -8,11 +8,11 @@ import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.types.VersionState; import io.apicurio.registry.utils.IoUtil; +import io.apicurio.registry.utils.impexp.EntityWriter; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; import io.apicurio.registry.utils.impexp.v3.BranchEntity; import io.apicurio.registry.utils.impexp.v3.ContentEntity; -import io.apicurio.registry.utils.impexp.v3.EntityWriter; import io.apicurio.tests.ApicurioRegistryBaseIT; import io.apicurio.tests.serdes.apicurio.JsonSchemaMsgFactory; import io.apicurio.tests.utils.Constants; diff --git a/integration-tests/src/test/java/io/apicurio/tests/migration/MigrationTestsDataInitializer.java b/integration-tests/src/test/java/io/apicurio/tests/migration/MigrationTestsDataInitializer.java index 746205e364..02478f0894 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/migration/MigrationTestsDataInitializer.java +++ b/integration-tests/src/test/java/io/apicurio/tests/migration/MigrationTestsDataInitializer.java @@ -8,9 +8,9 @@ import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.types.VersionState; import io.apicurio.registry.utils.IoUtil; +import io.apicurio.registry.utils.impexp.EntityWriter; import io.apicurio.registry.utils.impexp.v2.ContentEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; -import io.apicurio.registry.utils.impexp.v3.EntityWriter; import io.apicurio.registry.utils.tests.TestUtils; import io.apicurio.tests.serdes.apicurio.AvroGenericRecordSchemaFactory; import io.apicurio.tests.serdes.apicurio.JsonSchemaMsgFactory; @@ -46,6 +46,7 @@ public static void initializeMigrateTest(io.apicurio.registry.rest.client.v2.Reg migrateReferencesMap = new HashMap<>(); JsonSchemaMsgFactory jsonSchema = new JsonSchemaMsgFactory(); + // Create 50 artifacts in the default group. for (int idx = 0; idx < 50; idx++) { String artifactId = idx + "-" + UUID.randomUUID().toString(); @@ -58,6 +59,7 @@ public static void initializeMigrateTest(io.apicurio.registry.rest.client.v2.Reg migrateGlobalIds.add(response.getGlobalId()); } + // Create 15 artifacts in the "migrateTest" group for (int idx = 0; idx < 15; idx++) { AvroGenericRecordSchemaFactory avroSchema = new AvroGenericRecordSchemaFactory( List.of("a" + idx)); @@ -101,15 +103,18 @@ public static void initializeMigrateTest(io.apicurio.registry.rest.client.v2.Reg migrateGlobalIds.add(vmd.getGlobalId()); } + // Set an artifact rule. io.apicurio.registry.rest.client.v2.models.Rule createRule = new Rule(); createRule.setType(io.apicurio.registry.rest.client.v2.models.RuleType.VALIDITY); createRule.setConfig("SYNTAX_ONLY"); source.groups().byGroupId("migrateTest").artifacts().byArtifactId("avro-0").rules().post(createRule); + // Set a global compatibility rule. createRule.setType(RuleType.COMPATIBILITY); createRule.setConfig("BACKWARD"); source.admin().rules().post(createRule); + // Export the data to a ZIP file. var downloadHref = source.admin().export().get().getHref(); OkHttpClient client = new OkHttpClient(); DataMigrationIT.migrateDataToImport = client diff --git a/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/Export.java b/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/Export.java index 27e80e8f16..21f76491a8 100644 --- a/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/Export.java +++ b/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/Export.java @@ -7,13 +7,13 @@ import io.apicurio.registry.types.VersionState; import io.apicurio.registry.utils.IoUtil; import io.apicurio.registry.utils.export.mappers.ArtifactReferenceMapper; +import io.apicurio.registry.utils.impexp.EntityWriter; +import io.apicurio.registry.utils.impexp.ManifestEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; import io.apicurio.registry.utils.impexp.v3.ContentEntity; -import io.apicurio.registry.utils.impexp.v3.EntityWriter; import io.apicurio.registry.utils.impexp.v3.GlobalRuleEntity; -import io.apicurio.registry.utils.impexp.v3.ManifestEntity; import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.SchemaMetadata; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; diff --git a/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/ExportContext.java b/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/ExportContext.java index 189c0678f9..c18a7db725 100644 --- a/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/ExportContext.java +++ b/utils/exportConfluent/src/main/java/io/apicurio/registry/utils/export/ExportContext.java @@ -1,6 +1,6 @@ package io.apicurio.registry.utils.export; -import io.apicurio.registry.utils.impexp.v3.EntityWriter; +import io.apicurio.registry.utils.impexp.EntityWriter; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.rest.RestService; diff --git a/utils/importexport/pom.xml b/utils/importexport/pom.xml index 607a9fe6a8..8c2f4d46e4 100644 --- a/utils/importexport/pom.xml +++ b/utils/importexport/pom.xml @@ -28,6 +28,16 @@ jackson-databind + + com.fasterxml.jackson.core + jackson-databind + + + + commons-io + commons-io + + io.quarkus quarkus-core diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInfo.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInfo.java new file mode 100644 index 0000000000..c7a1db6a6c --- /dev/null +++ b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInfo.java @@ -0,0 +1,29 @@ +package io.apicurio.registry.utils.impexp; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.io.File; + +@NoArgsConstructor +@AllArgsConstructor +@Builder +@Getter +@Setter +@EqualsAndHashCode +@ToString +public class EntityInfo { + + private String path; + private EntityType type; + + public File toFile() { + return new File(path); + } + +} diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStream.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStream.java new file mode 100644 index 0000000000..71da779aae --- /dev/null +++ b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStream.java @@ -0,0 +1,11 @@ +package io.apicurio.registry.utils.impexp; + +import java.io.IOException; + +public interface EntityInputStream { + + /** + * Get the next import entity from the stream of entities being imported. + */ + Entity nextEntity() throws IOException; +} diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStreamImpl.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStreamImpl.java new file mode 100644 index 0000000000..e7a8f10cd3 --- /dev/null +++ b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityInputStreamImpl.java @@ -0,0 +1,16 @@ +package io.apicurio.registry.utils.impexp; + +import lombok.AllArgsConstructor; + +import java.io.IOException; + +@AllArgsConstructor +public class EntityInputStreamImpl implements EntityInputStream { + + private EntityReader reader; + + @Override + public Entity nextEntity() throws IOException { + return reader.readNextEntity(); + } +} diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityReader.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityReader.java new file mode 100644 index 0000000000..d2451f0be9 --- /dev/null +++ b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityReader.java @@ -0,0 +1,272 @@ +package io.apicurio.registry.utils.impexp; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.apicurio.registry.utils.IoUtil; +import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; +import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; +import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; +import io.apicurio.registry.utils.impexp.v3.BranchEntity; +import io.apicurio.registry.utils.impexp.v3.CommentEntity; +import io.apicurio.registry.utils.impexp.v3.ContentEntity; +import io.apicurio.registry.utils.impexp.v3.GlobalRuleEntity; +import io.apicurio.registry.utils.impexp.v3.GroupEntity; +import io.apicurio.registry.utils.impexp.v3.GroupRuleEntity; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +public class EntityReader { + + private static final ObjectMapper mapper; + static { + JsonFactory jsonFactory = new JsonFactory(); + jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); + mapper = new ObjectMapper(jsonFactory); + } + + private final transient Path importDirectory; + private List entities = null; + private int majorVersion = 3; + private int currentIndex = 0; + + /** + * Constructor. + */ + public EntityReader(Path importDirectory) { + this.importDirectory = importDirectory; + } + + public Entity readNextEntity() throws IOException { + if (entities == null) { + createEntityIndex(); + } + + EntityInfo entityInfo = getNextEntityInfo(); + if (entityInfo != null) { + EntityType entityType = entityInfo.getType(); + if (entityType != null) { + switch (entityType) { + case Artifact: + return readArtifact(entityInfo); + case ArtifactRule: + return readArtifactRule(entityInfo); + case ArtifactVersion: + return readArtifactVersion(entityInfo); + case Content: + return readContent(entityInfo); + case GlobalRule: + return readGlobalRule(entityInfo); + case Group: + return readGroup(entityInfo); + case GroupRule: + return readGroupRule(entityInfo); + case Comment: + return readComment(entityInfo); + case Branch: + return readBranch(entityInfo); + case Manifest: + return readManifest(entityInfo); + } + } + } + + return null; + } + + /** + * Return the next entity info object in the list. + */ + private EntityInfo getNextEntityInfo() { + if (entities == null) { + return null; + } + if (currentIndex >= entities.size()) { + return null; + } + return entities.get(currentIndex++); + } + + /** + * Create the index of entities found in the ZIP file. Make sure we read the entities in the following + * order:
+ *
    + *
  1. Manifest
    2. + *
  2. Rules
    3. + *
  3. Content
    4. + *
  4. Groups
    5. + *
  5. Group Rules
    6. + *
  6. Artifact Rules
    7. + *
  7. Artifact Versions
    8. + *
  8. Artifact Branches
    9. + *
  9. Comments
    10. + *
+ */ + private void createEntityIndex() { + EntityInfo manifest = null; + List rules = new LinkedList<>(); + List content = new LinkedList<>(); + List groups = new LinkedList<>(); + List groupRules = new LinkedList<>(); + List artifacts = new LinkedList<>(); + List artifactRules = new LinkedList<>(); + List versions = new LinkedList<>(); + List branches = new LinkedList<>(); + List comments = new LinkedList<>(); + + Collection allFiles = FileUtils.listFiles(this.importDirectory.toFile(), + new String[] { "json" }, true); + for (File file : allFiles) { + String path = file.getAbsolutePath(); + EntityType type = parseEntityType(path); + EntityInfo entityInfo = new EntityInfo(path, type); + switch (type) { + case Artifact: + artifacts.add(entityInfo); + break; + case ArtifactRule: + artifactRules.add(entityInfo); + break; + case ArtifactVersion: + versions.add(entityInfo); + break; + case Content: + content.add(entityInfo); + break; + case GlobalRule: + rules.add(entityInfo); + break; + case Group: + groups.add(entityInfo); + break; + case GroupRule: + groupRules.add(entityInfo); + break; + case Comment: + comments.add(entityInfo); + break; + case Branch: + branches.add(entityInfo); + break; + case Manifest: + manifest = entityInfo; + } + } + + entities = new LinkedList<>(); + if (manifest != null) { + entities.add(manifest); + } + + entities.addAll(rules); + entities.addAll(content); + entities.addAll(groups); + entities.addAll(groupRules); + entities.addAll(artifacts); + entities.addAll(versions); + entities.addAll(artifactRules); + entities.addAll(branches); + entities.addAll(comments); + } + + private Entity readContent(EntityInfo entry) throws IOException { + String path = entry.getPath(); + + if (majorVersion == 3) { + // Read the content entity from the *.Content.json file. + ContentEntity entity = this.readEntry(entry, ContentEntity.class); + + // Read the data for the content entity from the corresponding *.Content.data file. + String dataFilePath = path.replace(".json", ".data"); + File dataFile = new File(dataFilePath); + if (dataFile.exists() && dataFile.isFile()) { + entity.contentBytes = IoUtil.toBytes(dataFile); + } + return entity; + } else { + // Read the content entity from the *.Content.json file. + io.apicurio.registry.utils.impexp.v2.ContentEntity entity = this.readEntry(entry, + io.apicurio.registry.utils.impexp.v2.ContentEntity.class); + + // Read the data for the content entity from the corresponding *.Content.data file. + String dataFilePath = path.replace(".json", ".data"); + File dataFile = new File(dataFilePath); + if (dataFile.exists() && dataFile.isFile()) { + entity.contentBytes = IoUtil.toBytes(dataFile); + } + return entity; + } + } + + private ManifestEntity readManifest(EntityInfo entry) throws IOException { + ManifestEntity manifestEntity = readEntry(entry, ManifestEntity.class); + if (manifestEntity.systemVersion.startsWith("1")) { + this.majorVersion = 1; + } + if (manifestEntity.systemVersion.startsWith("2")) { + this.majorVersion = 2; + } + return manifestEntity; + } + + private Entity readGroup(EntityInfo entry) throws IOException { + return majorVersion == 3 ? readEntry(entry, GroupEntity.class) + : readEntry(entry, io.apicurio.registry.utils.impexp.v2.GroupEntity.class); + } + + private Entity readGroupRule(EntityInfo entry) throws IOException { + return readEntry(entry, GroupRuleEntity.class); + } + + private Entity readArtifact(EntityInfo entry) throws IOException { + return readEntry(entry, ArtifactEntity.class); + } + + private Entity readArtifactVersion(EntityInfo entry) throws IOException { + return majorVersion == 3 ? readEntry(entry, ArtifactVersionEntity.class) + : readEntry(entry, io.apicurio.registry.utils.impexp.v2.ArtifactVersionEntity.class); + } + + private Entity readArtifactRule(EntityInfo entry) throws IOException { + return majorVersion == 3 ? readEntry(entry, ArtifactRuleEntity.class) + : readEntry(entry, io.apicurio.registry.utils.impexp.v2.ArtifactRuleEntity.class); + } + + private Entity readComment(EntityInfo entry) throws IOException { + return majorVersion == 3 ? readEntry(entry, CommentEntity.class) + : readEntry(entry, io.apicurio.registry.utils.impexp.v2.CommentEntity.class); + } + + private Entity readBranch(EntityInfo entry) throws IOException { + return readEntry(entry, BranchEntity.class); + } + + private Entity readGlobalRule(EntityInfo entry) throws IOException { + return majorVersion == 3 ? readEntry(entry, GlobalRuleEntity.class) + : readEntry(entry, io.apicurio.registry.utils.impexp.v2.GlobalRuleEntity.class); + } + + private EntityType parseEntityType(String path) { + String[] split = path.split("\\."); + if (split.length > 2) { + String typeStr = split[split.length - 2]; + EntityType type = EntityType.valueOf(typeStr); + return type; + } + return null; + } + + private T readEntry(EntityInfo entry, Class theClass) throws IOException { + File file = entry.toFile(); + byte[] bytes = IoUtil.toBytes(file); + T entity = mapper.readerFor(theClass).readValue(bytes); + return entity; + } + +} diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/EntityWriter.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityWriter.java similarity index 84% rename from utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/EntityWriter.java rename to utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityWriter.java index dff6961efa..8be4019235 100644 --- a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/EntityWriter.java +++ b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/EntityWriter.java @@ -1,10 +1,17 @@ -package io.apicurio.registry.utils.impexp.v3; +package io.apicurio.registry.utils.impexp; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; -import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.EntityType; +import io.apicurio.registry.utils.impexp.v3.ArtifactEntity; +import io.apicurio.registry.utils.impexp.v3.ArtifactRuleEntity; +import io.apicurio.registry.utils.impexp.v3.ArtifactVersionEntity; +import io.apicurio.registry.utils.impexp.v3.BranchEntity; +import io.apicurio.registry.utils.impexp.v3.CommentEntity; +import io.apicurio.registry.utils.impexp.v3.ContentEntity; +import io.apicurio.registry.utils.impexp.v3.GlobalRuleEntity; +import io.apicurio.registry.utils.impexp.v3.GroupEntity; +import io.apicurio.registry.utils.impexp.v3.GroupRuleEntity; import java.io.IOException; import java.util.zip.ZipEntry; @@ -44,6 +51,9 @@ public void writeEntity(Entity entity) throws IOException { case Group: writeEntity((GroupEntity) entity); break; + case GroupRule: + writeEntity((GroupRuleEntity) entity); + break; case Artifact: writeEntity((ArtifactEntity) entity); break; @@ -94,6 +104,11 @@ private void writeEntity(GroupEntity entity) throws IOException { write(mdEntry, entity, GroupEntity.class); } + private void writeEntity(GroupRuleEntity entity) throws IOException { + ZipEntry mdEntry = createZipEntry(EntityType.GroupRule, entity.groupId, entity.type.name(), "json"); + write(mdEntry, entity, GroupRuleEntity.class); + } + private void writeEntity(ArtifactEntity entity) throws IOException { ZipEntry mdEntry = createZipEntry(EntityType.Artifact, entity.groupId, entity.artifactId, "MetaData", "json"); @@ -133,6 +148,10 @@ private ZipEntry createZipEntry(EntityType type, String fileName, String fileExt return createZipEntry(type, null, null, fileName, fileExt); } + private ZipEntry createZipEntry(EntityType type, String groupId, String fileName, String fileExt) { + return createZipEntry(type, groupId, null, fileName, fileExt); + } + private ZipEntry createZipEntry(EntityType type, String groupId, String artifactId, String fileName, String fileExt) { // TODO encode groupId, artifactId, and filename as path elements @@ -163,6 +182,10 @@ private ZipEntry createZipEntry(EntityType type, String groupId, String artifact case Group: path = String.format("groups/%s.%s.%s", fileName, type.name(), fileExt); break; + case GroupRule: + path = String.format("groups/%s/rules/%s.%s.%s", groupOrDefault(groupId), fileName, + type.name(), fileExt); + break; case Comment: path = String.format("comments/%s.%s.%s", fileName, type.name(), fileExt); break; diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/ManifestEntity.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/ManifestEntity.java similarity index 71% rename from utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/ManifestEntity.java rename to utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/ManifestEntity.java index a1499bf508..4e0e0c3649 100644 --- a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/ManifestEntity.java +++ b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/ManifestEntity.java @@ -1,7 +1,5 @@ -package io.apicurio.registry.utils.impexp.v3; +package io.apicurio.registry.utils.impexp; -import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.EntityType; import io.quarkus.runtime.annotations.RegisterForReflection; import java.util.Date; @@ -12,7 +10,7 @@ public class ManifestEntity extends Entity { public String systemVersion; public String systemName; public String systemDescription; - public String exportVersion = "1.0"; + public String exportVersion = "3.0"; public Date exportedOn = new Date(); public String exportedBy; diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityReader.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityReader.java deleted file mode 100644 index c0a66fabef..0000000000 --- a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityReader.java +++ /dev/null @@ -1,129 +0,0 @@ -package io.apicurio.registry.utils.impexp.v2; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.apicurio.registry.utils.IoUtil; -import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.EntityType; - -import java.io.IOException; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -public class EntityReader { - - private static final ObjectMapper mapper; - static { - JsonFactory jsonFactory = new JsonFactory(); - jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); - mapper = new ObjectMapper(jsonFactory); - } - - private final transient ZipInputStream zip; - - /** - * Constructor. - * - * @param zip - */ - public EntityReader(ZipInputStream zip) { - this.zip = zip; - } - - public Entity readEntity() throws IOException { - ZipEntry entry = zip.getNextEntry(); - while (entry != null && entry.isDirectory()) { - entry = zip.getNextEntry(); - } - if (entry != null) { - String path = entry.getName(); - EntityType entityType = parseEntityType(path); - if (entityType != null) { - switch (entityType) { - case ArtifactRule: - return readArtifactRule(entry); - case ArtifactVersion: - return readArtifactVersion(entry); - case Content: - return readContent(entry); - case GlobalRule: - return readGlobalRule(entry); - case Group: - return readGroup(entry); - case Comment: - return readComment(entry); - case Manifest: - return readManifest(entry); - } - } - } - - return null; - } - - private io.apicurio.registry.utils.impexp.v2.ContentEntity readContent(ZipEntry entry) - throws IOException { - if (entry.getName().endsWith(".json")) { - io.apicurio.registry.utils.impexp.v2.ContentEntity entity = this.readEntry(entry, - ContentEntity.class); - - ZipEntry dataEntry = zip.getNextEntry(); - if (!dataEntry.getName().endsWith(".Content.data")) { - // TODO what to do if this isn't the file we expect?? - } - - entity.contentBytes = IoUtil.toBytes(zip, false); - zip.read(entity.contentBytes); - return entity; - } else { - throw new IOException("Not yet supported: found .Content.data file before .Content.json"); - } - } - - private io.apicurio.registry.utils.impexp.v2.ManifestEntity readManifest(ZipEntry entry) - throws IOException { - return readEntry(entry, ManifestEntity.class); - } - - private io.apicurio.registry.utils.impexp.v2.GroupEntity readGroup(ZipEntry entry) throws IOException { - return readEntry(entry, GroupEntity.class); - } - - private io.apicurio.registry.utils.impexp.v2.ArtifactVersionEntity readArtifactVersion(ZipEntry entry) - throws IOException { - return this.readEntry(entry, ArtifactVersionEntity.class); - } - - private io.apicurio.registry.utils.impexp.v2.ArtifactRuleEntity readArtifactRule(ZipEntry entry) - throws IOException { - return this.readEntry(entry, ArtifactRuleEntity.class); - } - - private io.apicurio.registry.utils.impexp.v2.CommentEntity readComment(ZipEntry entry) - throws IOException { - return this.readEntry(entry, CommentEntity.class); - } - - private io.apicurio.registry.utils.impexp.v2.GlobalRuleEntity readGlobalRule(ZipEntry entry) - throws IOException { - return this.readEntry(entry, GlobalRuleEntity.class); - } - - private EntityType parseEntityType(String path) { - String[] split = path.split("\\."); - if (split.length > 2) { - String typeStr = split[split.length - 2]; - EntityType type = EntityType.valueOf(typeStr); - return type; - } - return null; - } - - private T readEntry(ZipEntry entry, Class theClass) throws IOException { - byte[] bytes = IoUtil.toBytes(zip, false); - T entity = mapper.readerFor(theClass).readValue(bytes); - return entity; - } - -} diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityType.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityType.java deleted file mode 100644 index 177b2d7af0..0000000000 --- a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/EntityType.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2021 Red Hat - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.apicurio.registry.utils.impexp.v2; - -import io.quarkus.runtime.annotations.RegisterForReflection; - -/** - * @author eric.wittmann@gmail.com - */ -@RegisterForReflection -public enum EntityType { - - Manifest, GlobalRule, Content, Group, ArtifactVersion, ArtifactRule, Comment - -} diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/ManifestEntity.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/ManifestEntity.java deleted file mode 100644 index b503e92861..0000000000 --- a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v2/ManifestEntity.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2021 Red Hat - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.apicurio.registry.utils.impexp.v2; - -import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.EntityType; -import io.quarkus.runtime.annotations.RegisterForReflection; - -import java.util.Date; - -/** - * @author eric.wittmann@gmail.com - */ -@RegisterForReflection -public class ManifestEntity extends Entity { - - public String systemVersion; - public String systemName; - public String systemDescription; - public String exportVersion = "1.0"; - public Date exportedOn = new Date(); - public String exportedBy; - - /** - * @see Entity#getEntityType() - */ - @Override - public EntityType getEntityType() { - return EntityType.Manifest; - } -} diff --git a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/EntityReader.java b/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/EntityReader.java deleted file mode 100644 index ecb79f8f44..0000000000 --- a/utils/importexport/src/main/java/io/apicurio/registry/utils/impexp/v3/EntityReader.java +++ /dev/null @@ -1,134 +0,0 @@ -package io.apicurio.registry.utils.impexp.v3; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.apicurio.registry.utils.IoUtil; -import io.apicurio.registry.utils.impexp.Entity; -import io.apicurio.registry.utils.impexp.EntityType; - -import java.io.IOException; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -public class EntityReader { - - private static final ObjectMapper mapper; - static { - JsonFactory jsonFactory = new JsonFactory(); - jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_TARGET, false); - mapper = new ObjectMapper(jsonFactory); - } - - private final transient ZipInputStream zip; - - /** - * Constructor. - * - * @param zip - */ - public EntityReader(ZipInputStream zip) { - this.zip = zip; - } - - public Entity readEntity() throws IOException { - ZipEntry entry = zip.getNextEntry(); - while (entry != null && entry.isDirectory()) { - entry = zip.getNextEntry(); - } - if (entry != null) { - String path = entry.getName(); - EntityType entityType = parseEntityType(path); - if (entityType != null) { - switch (entityType) { - case Artifact: - return readArtifact(entry); - case ArtifactRule: - return readArtifactRule(entry); - case ArtifactVersion: - return readArtifactVersion(entry); - case Content: - return readContent(entry); - case GlobalRule: - return readGlobalRule(entry); - case Group: - return readGroup(entry); - case Comment: - return readComment(entry); - case Branch: - return readBranch(entry); - case Manifest: - return readManifest(entry); - } - } - } - - return null; - } - - private ContentEntity readContent(ZipEntry entry) throws IOException { - if (entry.getName().endsWith(".json")) { - ContentEntity entity = this.readEntry(entry, ContentEntity.class); - - ZipEntry dataEntry = zip.getNextEntry(); - if (!dataEntry.getName().endsWith(".Content.data")) { - // TODO what to do if this isn't the file we expect?? - } - - entity.contentBytes = IoUtil.toBytes(zip, false); - zip.read(entity.contentBytes); - return entity; - } else { - throw new IOException("Not yet supported: found .Content.data file before .Content.json"); - } - } - - private ManifestEntity readManifest(ZipEntry entry) throws IOException { - return readEntry(entry, ManifestEntity.class); - } - - private GroupEntity readGroup(ZipEntry entry) throws IOException { - return readEntry(entry, GroupEntity.class); - } - - private ArtifactEntity readArtifact(ZipEntry entry) throws IOException { - return this.readEntry(entry, ArtifactEntity.class); - } - - private ArtifactVersionEntity readArtifactVersion(ZipEntry entry) throws IOException { - return this.readEntry(entry, ArtifactVersionEntity.class); - } - - private ArtifactRuleEntity readArtifactRule(ZipEntry entry) throws IOException { - return this.readEntry(entry, ArtifactRuleEntity.class); - } - - private CommentEntity readComment(ZipEntry entry) throws IOException { - return this.readEntry(entry, CommentEntity.class); - } - - private BranchEntity readBranch(ZipEntry entry) throws IOException { - return this.readEntry(entry, BranchEntity.class); - } - - private GlobalRuleEntity readGlobalRule(ZipEntry entry) throws IOException { - return this.readEntry(entry, GlobalRuleEntity.class); - } - - private EntityType parseEntityType(String path) { - String[] split = path.split("\\."); - if (split.length > 2) { - String typeStr = split[split.length - 2]; - EntityType type = EntityType.valueOf(typeStr); - return type; - } - return null; - } - - private T readEntry(ZipEntry entry, Class theClass) throws IOException { - byte[] bytes = IoUtil.toBytes(zip, false); - T entity = mapper.readerFor(theClass).readValue(bytes); - return entity; - } - -}