feat(REST API): Added labels to all search results (#5385)

* Added labels to all search results. The # of labels is capped to avoid overly large REST responses * Fixed issues with labels on branches (branches do not have labels).
Apicurio · Oct 23, 2024 · f73284d · f73284d
1 parent a5928c9
commit f73284d
Show file tree

Hide file tree

Showing 21 changed files with 332 additions and 8 deletions.
diff --git a/app/src/main/java/io/apicurio/registry/rest/RestConfig.java b/app/src/main/java/io/apicurio/registry/rest/RestConfig.java
@@ -14,6 +14,10 @@ public class RestConfig {
     @Info(category = "rest", description = "Max size of the artifact allowed to be downloaded from URL", availableSince = "2.2.6")
     int downloadMaxSize;
 
+    @ConfigProperty(name = "apicurio.rest.search-results.labels.max-size.bytes", defaultValue = "512")
+    @Info(category = "rest", description = "Max size of the labels (in bytes) per item from within search results", availableSince = "3.0.3")
+    int labelsInSearchResultsMaxSize;
+
     @ConfigProperty(name = "apicurio.rest.artifact.download.ssl-validation.disabled", defaultValue = "false")
     @Info(category = "rest", description = "Skip SSL validation when downloading artifacts from URL", availableSince = "2.2.6")
     boolean downloadSkipSSLValidation;
@@ -42,6 +46,10 @@ public int getDownloadMaxSize() {
         return this.downloadMaxSize;
     }
 
+    public int getLabelsInSearchResultsMaxSize() {
+        return this.labelsInSearchResultsMaxSize;
+    }
+
     public boolean getDownloadSkipSSLValidation() {
         return this.downloadSkipSSLValidation;
     }

diff --git a/app/src/main/java/io/apicurio/registry/rest/v3/V3ApiUtil.java b/app/src/main/java/io/apicurio/registry/rest/v3/V3ApiUtil.java
@@ -144,6 +144,7 @@ public static ArtifactSearchResults dtoToSearchResults(ArtifactSearchResultsDto
             sa.setModifiedOn(artifact.getModifiedOn());
             sa.setName(artifact.getName());
             sa.setArtifactType(artifact.getArtifactType());
+            sa.setLabels(artifact.getLabels());
             results.getArtifacts().add(sa);
         });
         return results;
@@ -161,6 +162,7 @@ public static GroupSearchResults dtoToSearchResults(GroupSearchResultsDto dto) {
             sg.setGroupId(group.getId());
             sg.setModifiedBy(group.getModifiedBy());
             sg.setModifiedOn(group.getModifiedOn());
+            sg.setLabels(group.getLabels());
             results.getGroups().add(sg);
         });
         return results;
@@ -203,6 +205,7 @@ public static VersionSearchResults dtoToSearchResults(VersionSearchResultsDto dt
             sv.setName(version.getName());
             sv.setState(version.getState());
             sv.setArtifactType(version.getArtifactType());
+            sv.setLabels(version.getLabels());
             results.getVersions().add(sv);
         });
         return results;

diff --git a/app/src/main/java/io/apicurio/registry/storage/dto/SearchedArtifactDto.java b/app/src/main/java/io/apicurio/registry/storage/dto/SearchedArtifactDto.java
@@ -9,6 +9,7 @@
 import lombok.ToString;
 
 import java.util.Date;
+import java.util.Map;
 
 @NoArgsConstructor
 @AllArgsConstructor
@@ -28,4 +29,6 @@ public class SearchedArtifactDto {
     private String artifactType;
     private Date modifiedOn;
     private String modifiedBy;
+    private Map<String, String> labels;
+
 }
diff --git a/app/src/main/java/io/apicurio/registry/storage/dto/SearchedGroupDto.java b/app/src/main/java/io/apicurio/registry/storage/dto/SearchedGroupDto.java
@@ -3,6 +3,7 @@
 import lombok.*;
 
 import java.util.Date;
+import java.util.Map;
 
 @NoArgsConstructor
 @AllArgsConstructor
@@ -19,4 +20,6 @@ public class SearchedGroupDto {
     private String owner;
     private Date modifiedOn;
     private String modifiedBy;
+    private Map<String, String> labels;
+
 }
diff --git a/app/src/main/java/io/apicurio/registry/storage/dto/SearchedVersionDto.java b/app/src/main/java/io/apicurio/registry/storage/dto/SearchedVersionDto.java
@@ -10,6 +10,7 @@
 import lombok.ToString;
 
 import java.util.Date;
+import java.util.Map;
 
 @NoArgsConstructor
 @AllArgsConstructor
@@ -34,4 +35,6 @@ public class SearchedVersionDto {
     private long globalId;
     private long contentId;
     private int versionOrder;
+    private Map<String, String> labels;
+
 }
diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/AbstractSqlRegistryStorage.java
@@ -23,6 +23,7 @@
 import io.apicurio.registry.model.GA;
 import io.apicurio.registry.model.GAV;
 import io.apicurio.registry.model.VersionId;
+import io.apicurio.registry.rest.RestConfig;
 import io.apicurio.registry.rules.compatibility.CompatibilityLevel;
 import io.apicurio.registry.rules.integrity.IntegrityLevel;
 import io.apicurio.registry.rules.validity.ValidityLevel;
@@ -219,6 +220,9 @@ public abstract class AbstractSqlRegistryStorage implements RegistryStorage {
     @Inject
     SemVerConfigProperties semVerConfigProps;
 
+    @Inject
+    RestConfig restConfig;
+
     @Inject
 
     protected SqlStatements sqlStatements() {
@@ -1112,6 +1116,7 @@ public ArtifactSearchResultsDto searchArtifacts(Set<SearchFilter> filters, Order
 
             // Execute artifact query
             List<SearchedArtifactDto> artifacts = artifactsQuery.map(SearchedArtifactMapper.instance).list();
+            limitReturnedLabelsInArtifacts(artifacts);
             // Execute count query
             Integer count = countQuery.mapTo(Integer.class).one();
 
@@ -1736,6 +1741,7 @@ public VersionSearchResultsDto searchVersions(Set<SearchFilter> filters, OrderBy
 
             // Execute query
             List<SearchedVersionDto> versions = versionsQuery.map(SearchedVersionMapper.instance).list();
+            limitReturnedLabelsInVersions(versions);
             // Execute count query
             Integer count = countQuery.mapTo(Integer.class).one();
 
@@ -2907,6 +2913,8 @@ public GroupSearchResultsDto searchGroups(Set<SearchFilter> filters, OrderBy ord
 
             // Execute query
             List<SearchedGroupDto> groups = groupsQuery.map(SearchedGroupMapper.instance).list();
+            limitReturnedLabelsInGroups(groups);
+
             // Execute count query
             Integer count = countQuery.mapTo(Integer.class).one();
 
@@ -3539,6 +3547,7 @@ public VersionSearchResultsDto getBranchVersions(GA ga, BranchId branchId, int o
 
             // Execute query
             List<SearchedVersionDto> versions = versionsQuery.map(SearchedVersionMapper.instance).list();
+            limitReturnedLabelsInVersions(versions);
             // Execute count query
             Integer count = countQuery.mapTo(Integer.class).one();
 
@@ -3788,4 +3797,52 @@ private boolean isMssql() {
     private boolean isH2() {
         return sqlStatements.dbType().equals("h2");
     }
+
+    /*
+     * Ensures that only a reasonable number/size of labels for each item in the list are returned. This is to
+     * guard against an unexpectedly enormous response size to a REST API search operation.
+     */
+
+    private Map<String, String> limitReturnedLabels(Map<String, String> labels) {
+        int maxBytes = restConfig.getLabelsInSearchResultsMaxSize();
+        if (labels != null && !labels.isEmpty()) {
+            Map<String, String> cappedLabels = new HashMap<>();
+            int totalBytes = 0;
+            for (String key : labels.keySet()) {
+                if (totalBytes < maxBytes) {
+                    String value = labels.get(key);
+                    cappedLabels.put(key, value);
+                    totalBytes += key.length() + (value != null ? value.length() : 0);
+                }
+            }
+            return cappedLabels;
+        }
+
+        return null;
+    }
+
+    private void limitReturnedLabelsInGroups(List<SearchedGroupDto> groups) {
+        groups.forEach(group -> {
+            Map<String, String> labels = group.getLabels();
+            Map<String, String> cappedLabels = limitReturnedLabels(labels);
+            group.setLabels(cappedLabels);
+        });
+    }
+
+    private void limitReturnedLabelsInArtifacts(List<SearchedArtifactDto> artifacts) {
+        artifacts.forEach(artifact -> {
+            Map<String, String> labels = artifact.getLabels();
+            Map<String, String> cappedLabels = limitReturnedLabels(labels);
+            artifact.setLabels(cappedLabels);
+        });
+    }
+
+    private void limitReturnedLabelsInVersions(List<SearchedVersionDto> versions) {
+        versions.forEach(version -> {
+            Map<String, String> labels = version.getLabels();
+            Map<String, String> cappedLabels = limitReturnedLabels(labels);
+            version.setLabels(cappedLabels);
+        });
+    }
+
 }
diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedArtifactMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedArtifactMapper.java
@@ -32,6 +32,7 @@ public SearchedArtifactDto map(ResultSet rs) throws SQLException {
         dto.setModifiedBy(rs.getString("modifiedBy"));
         dto.setModifiedOn(rs.getTimestamp("modifiedOn"));
         dto.setArtifactType(rs.getString("type"));
+        dto.setLabels(RegistryContentUtils.deserializeLabels(rs.getString("labels")));
         return dto;
     }
 

diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedGroupMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedGroupMapper.java
@@ -1,6 +1,7 @@
 package io.apicurio.registry.storage.impl.sql.mappers;
 
 import io.apicurio.registry.storage.dto.SearchedGroupDto;
+import io.apicurio.registry.storage.impl.sql.RegistryContentUtils;
 import io.apicurio.registry.storage.impl.sql.jdb.RowMapper;
 
 import java.sql.ResultSet;
@@ -28,6 +29,7 @@ public SearchedGroupDto map(ResultSet rs) throws SQLException {
         dto.setDescription(rs.getString("description"));
         dto.setModifiedBy(rs.getString("modifiedBy"));
         dto.setModifiedOn(rs.getTimestamp("modifiedOn"));
+        dto.setLabels(RegistryContentUtils.deserializeLabels(rs.getString("labels")));
         return dto;
     }
 }
diff --git a/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedVersionMapper.java b/app/src/main/java/io/apicurio/registry/storage/impl/sql/mappers/SearchedVersionMapper.java
@@ -38,6 +38,7 @@ public SearchedVersionDto map(ResultSet rs) throws SQLException {
         dto.setName(rs.getString("name"));
         dto.setDescription(rs.getString("description"));
         dto.setArtifactType(rs.getString("type"));
+        dto.setLabels(RegistryContentUtils.deserializeLabels(rs.getString("labels")));
         return dto;
     }
 

diff --git a/app/src/main/resources/application.properties b/app/src/main/resources/application.properties
@@ -150,6 +150,7 @@ apicurio.redirects.root=/,/apis
 apicurio.rest.artifact.download.max-size.bytes=1000000
 apicurio.rest.artifact.download.ssl-validation.disabled=false
 apicurio.rest.artifact.deletion.enabled=false
+apicurio.rest.search-results.labels.max-size.bytes=512
 # Api date format
 apicurio.apis.date-format=yyyy-MM-dd'T'HH:mm:ss'Z'
 apicurio.apis.date-format-timezone=UTC

diff --git a/...src/test/java/io/apicurio/registry/noprofile/rest/v3/CappedLabelsInSearchResultsTest.java b/...src/test/java/io/apicurio/registry/noprofile/rest/v3/CappedLabelsInSearchResultsTest.java
@@ -0,0 +1,52 @@
+package io.apicurio.registry.noprofile.rest.v3;
+
+import io.apicurio.registry.AbstractResourceTestBase;
+import io.apicurio.registry.rest.client.models.CreateGroup;
+import io.apicurio.registry.rest.client.models.GroupMetaData;
+import io.apicurio.registry.rest.client.models.GroupSearchResults;
+import io.apicurio.registry.rest.client.models.Labels;
+import io.apicurio.registry.utils.tests.TestUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+
+@QuarkusTest
+public class CappedLabelsInSearchResultsTest extends AbstractResourceTestBase {
+
+    @Test
+    public void testCappedLabelsInGroupSearch() throws Exception {
+        String groupId = TestUtils.generateGroupId();
+
+        // Lots of labels! Too many labels.
+        Labels labels = new Labels();
+        labels.setAdditionalData(new HashMap<>());
+        for (int idx = 1000; idx < 1500; idx++) {
+            labels.getAdditionalData().put("test-key-" + idx, "test-value-" + idx);
+        }
+
+        // Create a group with all these labels.
+        CreateGroup createGroup = new CreateGroup();
+        createGroup.setGroupId(groupId);
+        createGroup.setLabels(labels);
+        clientV3.groups().post(createGroup);
+
+        // Get the group meta data
+        GroupMetaData gmd = clientV3.groups().byGroupId(groupId).get();
+        Assertions.assertNotNull(gmd);
+        Assertions.assertEquals(groupId, gmd.getGroupId());
+        Assertions.assertNotNull(gmd.getLabels());
+        Assertions.assertEquals(500, gmd.getLabels().getAdditionalData().size());
+
+        // Search for the group.
+        GroupSearchResults results = clientV3.search().groups().get(request -> {
+            request.queryParameters.groupId = groupId;
+        });
+        Assertions.assertEquals(1, results.getGroups().size());
+
+        Assertions.assertNotNull(results.getGroups().get(0).getLabels());
+        // Only 19 labels are returned due to the cap/limit enforced on returning labels in searches
+        Assertions.assertEquals(19, results.getGroups().get(0).getLabels().getAdditionalData().size());
+    }
+}
diff --git a/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/SearchArtifactsTest.java b/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/SearchArtifactsTest.java
@@ -1,10 +1,13 @@
 package io.apicurio.registry.noprofile.rest.v3;
 
 import io.apicurio.registry.AbstractResourceTestBase;
+import io.apicurio.registry.rest.client.models.ArtifactSearchResults;
 import io.apicurio.registry.rest.v3.beans.EditableArtifactMetaData;
 import io.apicurio.registry.types.ArtifactType;
 import io.apicurio.registry.types.ContentTypes;
+import io.apicurio.registry.utils.tests.TestUtils;
 import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
@@ -95,7 +98,7 @@ public void testSearchArtifactsByDescription() throws Exception {
 
     @Test
     public void testSearchArtifactsByLabels() throws Exception {
-        String group = UUID.randomUUID().toString();
+        String group = TestUtils.generateGroupId();
         String artifactContent = resourceToString("openapi-empty.json");
 
         // Create 5 artifacts with various labels
@@ -152,6 +155,19 @@ public void testSearchArtifactsByLabels() throws Exception {
                 .statusCode(400);
         given().when().queryParam("labels", "all-key:").get("/registry/v3/search/artifacts").then()
                 .statusCode(400);
+
+        // Test that search results contain the labels
+        ArtifactSearchResults results = clientV3.search().artifacts().get(conf -> {
+            conf.queryParameters.groupId = group;
+            conf.queryParameters.labels = new String[] { "key-1:value-1" };
+        });
+        Assertions.assertNotNull(results);
+        Assertions.assertEquals(1, results.getArtifacts().size());
+        Assertions.assertNotNull(results.getArtifacts().get(0).getLabels());
+        Assertions.assertEquals(
+                Map.of("key-1", "value-1", "another-key-1", "another-value-1", "all-key", "all-value",
+                        "a-key-1", "lorem ipsum", "extra-key-1", "lorem ipsum"),
+                results.getArtifacts().get(0).getLabels().getAdditionalData());
     }
 
     @Test

diff --git a/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/SearchGroupsTest.java b/app/src/test/java/io/apicurio/registry/noprofile/rest/v3/SearchGroupsTest.java
@@ -79,6 +79,10 @@ public void testSearchGroupsByLabels() throws Exception {
             request.queryParameters.groupId = groupId + "1";
         });
         Assertions.assertEquals(1, results.getGroups().size());
+        // Note: ensure that labels are returned in the search results
+        Assertions.assertNotNull(results.getGroups().get(0).getLabels());
+        Assertions.assertEquals(Map.of("byLabels", "byLabels-value-1", "byLabels-1", "byLabels-value-1"),
+                results.getGroups().get(0).getLabels().getAdditionalData());
 
         results = clientV3.search().groups().get(request -> {
             request.queryParameters.labels = new String[] { "byLabels" };
@@ -109,5 +113,4 @@ public void testSearchGroupsByLabels() throws Exception {
         Assertions.assertEquals(1, results.getGroups().size());
         Assertions.assertEquals("testSearchGroupsByLabels3", results.getGroups().get(0).getGroupId());
     }
-
 }