Skip to content

Commit

Permalink
Several improvements to the Import/Export process (#5037)
Browse files Browse the repository at this point in the history 
* Import now writes ZIP to disk first. All import options now support both v2 and v3 ZIP files.  Fails on others.

* By default, require a registry to be empty before allowing an import

* Add debugging to migration test

* code formatting

* Make sure to read artifact rules after versions so that importing from v2 works
  • Loading branch information
@EricWittmann
EricWittmann authored Aug 16, 2024
1 parent 6a58724 commit 5eec328
Show file tree
Hide file tree
Showing 49 changed files with 823 additions and 758 deletions.
55 changes: 19 additions & 36 deletions app/src/main/java/io/apicurio/registry/ImportLifecycleBean.java
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,22 @@
package io.apicurio.registry;

import io.apicurio.common.apps.config.Info;
import io.apicurio.registry.rest.ConflictException;
import io.apicurio.registry.rest.v3.AdminResourceImpl;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.StorageEvent;
import io.apicurio.registry.storage.StorageEventType;
import io.apicurio.registry.storage.error.ReadOnlyStorageException;
import io.apicurio.registry.storage.impexp.EntityInputStream;
import io.apicurio.registry.storage.importing.ImportExportConfigProperties;
import io.apicurio.registry.types.Current;
import io.apicurio.registry.utils.impexp.Entity;
import io.apicurio.registry.utils.impexp.v3.EntityReader;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.ObservesAsync;
import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.slf4j.Logger;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.zip.ZipInputStream;

@ApplicationScoped
public class ImportLifecycleBean {
Expand All @@ -33,42 +28,30 @@ public class ImportLifecycleBean {
@Current
RegistryStorage storage;

@ConfigProperty(name = "apicurio.import.url")
@Info(category = "import", description = "The import URL", availableSince = "2.1.0.Final")
Optional<URL> registryImportUrlProp;
@Inject
ImportExportConfigProperties importExportProps;

@Inject
AdminResourceImpl v3Admin;

void onStorageReady(@ObservesAsync StorageEvent ev) {
if (StorageEventType.READY.equals(ev.getType()) && registryImportUrlProp.isPresent()) {
if (StorageEventType.READY.equals(ev.getType())
&& importExportProps.registryImportUrlProp.isPresent()) {
log.info("Import URL exists.");
final URL registryImportUrl = registryImportUrlProp.get();
final URL registryImportUrl = importExportProps.registryImportUrlProp.get();
try (final InputStream registryImportZip = new BufferedInputStream(
registryImportUrl.openStream())) {
log.info("Importing {} on startup.", registryImportUrl);
final ZipInputStream zip = new ZipInputStream(registryImportZip, StandardCharsets.UTF_8);
final EntityReader reader = new EntityReader(zip);
try (EntityInputStream stream = new EntityInputStream() {
@Override
public Entity nextEntity() {
try {
return reader.readEntity();
} catch (Exception e) {
log.error("Error reading data from import ZIP file {}.", registryImportUrl, e);
return null;
}
}

@Override
public void close() throws IOException {
zip.close();
}
}) {
storage.importData(stream, true, true);
log.info("Registry successfully imported from {}", registryImportUrl);
} catch (ReadOnlyStorageException e) {
log.error("Registry import failed, because the storage is in read-only mode.");
}
v3Admin.importData(null, null, null, registryImportZip);
log.info("Registry successfully imported from {}", registryImportUrl);
} catch (IOException ioe) {
log.error("Registry import from {} failed", registryImportUrl, ioe);
} catch (ReadOnlyStorageException rose) {
log.error("Registry import failed, because the storage is in read-only mode.");
} catch (ConflictException ce) {
log.info("Import skipped, registry not empty.");
} catch (Exception e) {
log.error("Registry import failed", e);
}
}
}
Expand Down
71 changes: 9 additions & 62 deletions app/src/main/java/io/apicurio/registry/rest/v2/AdminResourceImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package io.apicurio.registry.rest.v2;

import io.apicurio.common.apps.config.Dynamic;
import io.apicurio.common.apps.config.DynamicConfigPropertyDef;
import io.apicurio.common.apps.config.DynamicConfigPropertyDto;
import io.apicurio.common.apps.config.DynamicConfigPropertyIndex;
import io.apicurio.common.apps.config.Info;
import io.apicurio.common.apps.logging.Logged;
import io.apicurio.common.apps.logging.audit.Audited;
import io.apicurio.registry.auth.Authorized;
Expand All @@ -16,51 +14,37 @@
import io.apicurio.registry.rest.MissingRequiredParameterException;
import io.apicurio.registry.rest.v2.beans.ArtifactTypeInfo;
import io.apicurio.registry.rest.v2.beans.ConfigurationProperty;
import io.apicurio.registry.rest.v2.beans.DownloadRef;
import io.apicurio.registry.rest.v2.beans.RoleMapping;
import io.apicurio.registry.rest.v2.beans.Rule;
import io.apicurio.registry.rest.v2.beans.UpdateConfigurationProperty;
import io.apicurio.registry.rest.v2.beans.UpdateRole;
import io.apicurio.registry.rest.v2.shared.DataExporter;
import io.apicurio.registry.rules.DefaultRuleDeletionException;
import io.apicurio.registry.rules.RulesProperties;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.dto.DownloadContextDto;
import io.apicurio.registry.storage.dto.DownloadContextType;
import io.apicurio.registry.storage.dto.RoleMappingDto;
import io.apicurio.registry.storage.dto.RuleConfigurationDto;
import io.apicurio.registry.storage.error.ConfigPropertyNotFoundException;
import io.apicurio.registry.storage.error.InvalidPropertyValueException;
import io.apicurio.registry.storage.error.RuleNotFoundException;
import io.apicurio.registry.storage.impexp.EntityInputStream;
import io.apicurio.registry.storage.importing.ImportExportConfigProperties;
import io.apicurio.registry.types.Current;
import io.apicurio.registry.types.RoleType;
import io.apicurio.registry.types.RuleType;
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
import io.apicurio.registry.utils.impexp.Entity;
import io.apicurio.registry.utils.impexp.v2.EntityReader;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.interceptor.Interceptors;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.eclipse.microprofile.config.Config;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.slf4j.Logger;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipInputStream;

import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_FOR_BROWSER;
import static io.apicurio.common.apps.logging.audit.AuditingConstants.KEY_NAME;
Expand Down Expand Up @@ -97,17 +81,12 @@ public class AdminResourceImpl implements AdminResource {
Config config;

@Inject
DataExporter exporter;
ImportExportConfigProperties importExportProps;

@Context
HttpServletRequest request;

@Dynamic(label = "Download link expiry", description = "The number of seconds that a generated link to a .zip download file is active before expiring.")
@ConfigProperty(name = "apicurio.download.href.ttl.seconds", defaultValue = "30")
@Info(category = "download", description = "Download link expiry", availableSince = "2.1.2.Final")
Supplier<Long> downloadHrefTtl;
@Inject
io.apicurio.registry.rest.v3.AdminResourceImpl v3Admin;

private static final void requireParameter(String parameterName, Object parameterValue) {
private static void requireParameter(String parameterName, Object parameterValue) {
if (parameterValue == null) {
throw new MissingRequiredParameterException(parameterName);
}
Expand All @@ -124,7 +103,6 @@ public List<ArtifactTypeInfo> listArtifactTypes() {
ati.setName(t);
return ati;
}).collect(Collectors.toList());

}

/**
Expand Down Expand Up @@ -246,26 +224,7 @@ public void deleteGlobalRule(RuleType rule) {
@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin)
public void importData(Boolean xRegistryPreserveGlobalId, Boolean xRegistryPreserveContentId,
InputStream data) {
final ZipInputStream zip = new ZipInputStream(data, StandardCharsets.UTF_8);
final EntityReader reader = new EntityReader(zip);
EntityInputStream stream = new EntityInputStream() {
@Override
public Entity nextEntity() throws IOException {
try {
return reader.readEntity();
} catch (Exception e) {
log.error("Error reading data from import ZIP file.", e);
return null;
}
}

@Override
public void close() throws IOException {
zip.close();
}
};
this.storage.upgradeData(stream, isNullOrTrue(xRegistryPreserveGlobalId),
isNullOrTrue(xRegistryPreserveContentId));
v3Admin.importData(xRegistryPreserveGlobalId, xRegistryPreserveContentId, false, data);
}

/**
Expand All @@ -275,20 +234,8 @@ public void close() throws IOException {
@Audited(extractParameters = { "0", KEY_FOR_BROWSER })
@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.Admin)
public Response exportData(Boolean forBrowser) {
String acceptHeader = request.getHeader("Accept");
if (Boolean.TRUE.equals(forBrowser) || MediaType.APPLICATION_JSON.equals(acceptHeader)) {
long expires = System.currentTimeMillis() + (downloadHrefTtl.get() * 1000);
DownloadContextDto downloadCtx = DownloadContextDto.builder().type(DownloadContextType.EXPORT)
.expires(expires).build();
String downloadId = storage.createDownload(downloadCtx);
String downloadHref = createDownloadHref(downloadId);
DownloadRef downloadRef = new DownloadRef();
downloadRef.setDownloadId(downloadId);
downloadRef.setHref(downloadHref);
return Response.ok(downloadRef).type(MediaType.APPLICATION_JSON_TYPE).build();
} else {
return exporter.exportData();
}
throw new UnsupportedOperationException(
"Exporting data using the Registry Core v2 API is no longer supported. Use the v3 API.");
}

/**
Expand Down Expand Up @@ -463,7 +410,7 @@ private ConfigurationProperty defToConfigurationProperty(DynamicConfigPropertyDe

/**
* Lookup the dynamic configuration property being set. Ensure that it exists (throws a
* {@link NotFoundException} if it does not.
* {@link io.apicurio.registry.storage.error.NotFoundException} if it does not.
*
* @param propertyName the name of the dynamic property
* @return the dynamic config property definition
Expand Down
20 changes: 0 additions & 20 deletions app/src/main/java/io/apicurio/registry/rest/v2/DownloadsResourceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,8 @@
import io.apicurio.registry.auth.AuthorizedStyle;
import io.apicurio.registry.metrics.health.liveness.ResponseErrorLivenessCheck;
import io.apicurio.registry.metrics.health.readiness.ResponseTimeoutReadinessCheck;
import io.apicurio.registry.rest.v2.shared.DataExporter;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.dto.DownloadContextDto;
import io.apicurio.registry.storage.dto.DownloadContextType;
import io.apicurio.registry.storage.error.DownloadNotFoundException;
import io.apicurio.registry.types.Current;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.interceptor.Interceptors;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
Expand All @@ -27,25 +21,11 @@
@Path("/apis/registry/v2/downloads")
public class DownloadsResourceImpl {

@Inject
@Current
RegistryStorage storage;

@Inject
DataExporter exporter;

@Authorized(style = AuthorizedStyle.None, level = AuthorizedLevel.None)
@GET
@Path("{downloadId}")
@Produces("*/*")
public Response download(@PathParam("downloadId") String downloadId) {
DownloadContextDto downloadContext = storage.consumeDownload(downloadId);
if (downloadContext.getType() == DownloadContextType.EXPORT) {
return exporter.exportData();
}

// TODO support other types of downloads (e.g. download content by contentId)

throw new DownloadNotFoundException();
}

Expand Down
61 changes: 0 additions & 61 deletions app/src/main/java/io/apicurio/registry/rest/v2/shared/DataExporter.java
View file

This file was deleted.

Loading

0 comments on commit 5eec328

Please sign in to comment.