diff --git a/app/pom.xml b/app/pom.xml index 215a2b04c6..6a80b4d93b 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -258,17 +258,17 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka test io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-protobuf-serde-kafka test io.apicurio - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-jsonschema-serde-kafka test diff --git a/app/src/test/java/io/apicurio/registry/noprofile/JsonSerdeTest.java b/app/src/test/java/io/apicurio/registry/noprofile/JsonSerdeTest.java index 2f563c0b01..9321d34876 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/JsonSerdeTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/JsonSerdeTest.java @@ -1,7 +1,9 @@ package io.apicurio.registry.noprofile; import io.apicurio.registry.AbstractResourceTestBase; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.resolver.SchemaResolverConfig; +import io.apicurio.registry.serde.config.KafkaSerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaDeserializer; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaSerializer; import io.apicurio.registry.support.Person; @@ -39,12 +41,11 @@ public void testSchema() throws Exception { Person person = new Person("Ales", "Justin", 23); - try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(clientV3, true); - JsonSchemaKafkaDeserializer deserializer = new JsonSchemaKafkaDeserializer<>(clientV3, - true)) { + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(clientV3); + JsonSchemaKafkaDeserializer deserializer = new JsonSchemaKafkaDeserializer<>(clientV3)) { - Map configs = Map.of(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId, - SerdeConfig.ENABLE_HEADERS, "true"); + Map configs = Map.of(SchemaResolverConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId, + KafkaSerdeConfig.ENABLE_HEADERS, "true", SerdeConfig.VALIDATION_ENABLED, "true"); serializer.configure(configs, false); deserializer.configure(configs, false); diff --git a/app/src/test/java/io/apicurio/registry/noprofile/serde/AvroSerdeTest.java b/app/src/test/java/io/apicurio/registry/noprofile/serde/AvroSerdeTest.java index 33e806f890..1ee4bc2587 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/serde/AvroSerdeTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/serde/AvroSerdeTest.java @@ -13,15 +13,12 @@ import io.apicurio.registry.AbstractResourceTestBase; import io.apicurio.registry.client.auth.VertXAuthFactory; import io.apicurio.registry.model.GroupId; -import io.apicurio.registry.resolver.SchemaResolverConfig; import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.rest.client.models.VersionMetaData; -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.SerdeHeaders; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.avro.AvroSerdeConfig; import io.apicurio.registry.serde.avro.DefaultAvroDatumProvider; import io.apicurio.registry.serde.avro.ReflectAllowNullAvroDatumProvider; import io.apicurio.registry.serde.avro.ReflectAvroDatumProvider; @@ -29,6 +26,9 @@ import io.apicurio.registry.serde.avro.strategy.RecordIdStrategy; import io.apicurio.registry.serde.avro.strategy.TopicRecordIdStrategy; import io.apicurio.registry.serde.config.IdOption; +import io.apicurio.registry.serde.config.KafkaSerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; +import io.apicurio.registry.serde.headers.KafkaSerdeHeaders; import io.apicurio.registry.support.Tester; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.types.ContentTypes; @@ -97,7 +97,7 @@ public void testConfiguration() throws Exception { config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); config.put(SerdeConfig.EXPLICIT_ARTIFACT_VERSION, "1"); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, TopicRecordIdStrategy.class.getName()); - config.put(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class.getName()); + config.put(AvroSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class.getName()); Serializer serializer = new AvroKafkaSerializer(); serializer.configure(config, true); @@ -117,8 +117,8 @@ public void testConfiguration() throws Exception { Assertions.assertEquals(record, deserializedRecord); Assertions.assertEquals("somebar", record.get("bar").toString()); - config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, TopicRecordIdStrategy.class); - config.put(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class); + config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, TopicRecordIdStrategy.class.getName()); + config.put(AvroSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class.getName()); serializer.configure(config, true); bytes = serializer.serialize(topic, record); @@ -127,12 +127,13 @@ record = deserializer.deserialize(topic, bytes); Assertions.assertEquals("somebar", record.get("bar").toString()); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, TopicRecordIdStrategy.class.getName()); - config.put(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class.getName()); + config.put(AvroSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class.getName()); serializer.configure(config, true); bytes = serializer.serialize(topic, record); deserializer.configure(deserializerConfig, true); record = deserializer.deserialize(topic, bytes); Assertions.assertEquals("somebar", record.get("bar").toString()); + }); serializer.close(); @@ -213,12 +214,12 @@ public void testAvroJSON() throws Exception { Deserializer deserializer = new AvroKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); serializer.configure(config, false); config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); deserializer.configure(config, false); GenericData.Record record = new GenericData.Record(schema); @@ -255,14 +256,13 @@ public void avroJsonWithReferences() throws Exception { Deserializer deserializer = new AvroKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); serializer.configure(config, false); config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); - config.putIfAbsent(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, - ReflectAvroDatumProvider.class.getName()); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); + config.putIfAbsent(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); deserializer.configure(config, false); AvroSchemaB avroSchemaB = new AvroSchemaB(); @@ -330,15 +330,14 @@ public void avroJsonWithReferencesDereferenced() throws Exception { Deserializer deserializer = new AvroKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); - config.put(SchemaResolverConfig.DEREFERENCE_SCHEMA, "true"); + config.put(SerdeConfig.DEREFERENCE_SCHEMA, "true"); serializer.configure(config, false); config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); - config.putIfAbsent(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, - ReflectAvroDatumProvider.class.getName()); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); + config.putIfAbsent(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); deserializer.configure(config, false); AvroSchemaB avroSchemaB = new AvroSchemaB(); @@ -407,15 +406,14 @@ public void avroJsonWithReferencesDeserializerDereferenced() throws Exception { Deserializer deserializer = new AvroKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); serializer.configure(config, false); config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); - config.putIfAbsent(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, - ReflectAvroDatumProvider.class.getName()); - config.putIfAbsent(SchemaResolverConfig.DESERIALIZER_DEREFERENCE_SCHEMA, "true"); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); + config.putIfAbsent(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); + config.putIfAbsent(SerdeConfig.DESERIALIZER_DEREFERENCE_SCHEMA, "true"); deserializer.configure(config, false); AvroSchemaB avroSchemaB = new AvroSchemaB(); @@ -480,14 +478,13 @@ public void issue4463Test() throws Exception { Deserializer deserializer = new AvroKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); serializer.configure(config, false); config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); - config.putIfAbsent(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, - ReflectAvroDatumProvider.class.getName()); + config.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); + config.putIfAbsent(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); deserializer.configure(config, false); LeadFallErstellen leadFallErstellen = LeadFallErstellen.newBuilder() @@ -526,12 +523,12 @@ public void testAvroUsingHeaders() throws Exception { Deserializer deserializer = new AvroKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); serializer.configure(config, false); config = new HashMap<>(); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); deserializer.configure(config, false); GenericData.Record record = new GenericData.Record(schema); @@ -541,8 +538,8 @@ public void testAvroUsingHeaders() throws Exception { Headers headers = new RecordHeaders(); byte[] bytes = serializer.serialize(artifactId, headers, record); - Assertions.assertNotNull(headers.lastHeader(SerdeHeaders.HEADER_VALUE_CONTENT_ID)); - headers.lastHeader(SerdeHeaders.HEADER_VALUE_CONTENT_ID); + Assertions.assertNotNull(headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_CONTENT_ID)); + headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_CONTENT_ID); GenericData.Record ir = deserializer.deserialize(artifactId, headers, bytes); @@ -564,13 +561,13 @@ public void testReferenceRaw() throws Exception { Deserializer deserializer = new AvroKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, RecordIdStrategy.class.getName()); serializer.configure(config, false); config = new HashMap<>(); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); deserializer.configure(config, false); GenericData.EnumSymbol record = new GenericData.EnumSymbol(eventTypeSchema, "UNDEFINED"); @@ -579,8 +576,8 @@ public void testReferenceRaw() throws Exception { Headers headers = new RecordHeaders(); byte[] bytes = serializer.serialize(artifactId, headers, record); - Assertions.assertNotNull(headers.lastHeader(SerdeHeaders.HEADER_VALUE_CONTENT_ID)); - Header contentId = headers.lastHeader(SerdeHeaders.HEADER_VALUE_CONTENT_ID); + Assertions.assertNotNull(headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_CONTENT_ID)); + Header contentId = headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_CONTENT_ID); long contentIdKey = ByteBuffer.wrap(contentId.value()).getLong(); waitForSchemaLongId(id -> { @@ -625,13 +622,12 @@ private void testAvroReflect(Class artifactResolverStrategyClass, Class da Map config = new HashMap<>(); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); - config.put(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, datumProvider.getName()); - config.put(SchemaResolverConfig.ARTIFACT_RESOLVER_STRATEGY, - artifactResolverStrategyClass.getName()); + config.put(AvroSerdeConfig.AVRO_DATUM_PROVIDER, datumProvider.getName()); + config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, artifactResolverStrategyClass.getName()); serializer.configure(config, false); config = new HashMap<>(); - config.put(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, datumProvider.getName()); + config.put(AvroSerdeConfig.AVRO_DATUM_PROVIDER, datumProvider.getName()); deserializer.configure(config, false); String artifactId = generateArtifactId(); diff --git a/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java b/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java index 38e1a64697..a39f73a2e1 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/serde/JsonSchemaSerdeTest.java @@ -16,9 +16,10 @@ import io.apicurio.registry.rest.client.models.IfArtifactExists; import io.apicurio.registry.rest.client.models.VersionMetaData; import io.apicurio.registry.serde.SchemaResolverConfigurer; -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.SerdeHeaders; import io.apicurio.registry.serde.config.IdOption; +import io.apicurio.registry.serde.config.KafkaSerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; +import io.apicurio.registry.serde.headers.KafkaSerdeHeaders; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaDeserializer; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaSerializer; import io.apicurio.registry.serde.jsonschema.JsonSchemaParser; @@ -82,22 +83,23 @@ public void testJsonSchemaSerde() throws Exception { Person person = new Person("Ales", "Justin", 23); - try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); + config.put(SerdeConfig.VALIDATION_ENABLED, "true"); serializer.configure(config, false); - deserializer.configure(Collections.singletonMap(SerdeConfig.ENABLE_HEADERS, "true"), false); + deserializer.configure(Collections.singletonMap(KafkaSerdeConfig.ENABLE_HEADERS, "true"), false); Headers headers = new RecordHeaders(); byte[] bytes = serializer.serialize(artifactId, headers, person); - Assertions.assertNotNull(headers.lastHeader(SerdeHeaders.HEADER_VALUE_CONTENT_ID)); - headers.lastHeader(SerdeHeaders.HEADER_VALUE_CONTENT_ID); + Assertions.assertNotNull(headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_CONTENT_ID)); + headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_CONTENT_ID); person = deserializer.deserialize(artifactId, headers, bytes); @@ -132,18 +134,18 @@ public void testJsonSchemaSerdeAutoRegister() throws Exception { Person person = new Person("Carles", "Arnal", 30); - try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); config.put(SerdeConfig.SCHEMA_LOCATION, "/io/apicurio/registry/util/json-schema.json"); config.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, true); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); serializer.configure(config, false); - deserializer.configure(Collections.singletonMap(SerdeConfig.ENABLE_HEADERS, "true"), false); + deserializer.configure(Collections.singletonMap(KafkaSerdeConfig.ENABLE_HEADERS, "true"), false); Headers headers = new RecordHeaders(); byte[] bytes = serializer.serialize(artifactId, headers, person); @@ -188,31 +190,31 @@ public void testJsonSchemaSerdeHeaders() throws Exception { Person person = new Person("Ales", "Justin", 23); - try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); + config.put(SerdeConfig.VALIDATION_ENABLED, "true"); config.put(SerdeConfig.USE_ID, IdOption.globalId.name()); serializer.configure(config, false); - deserializer.configure( - Map.of(SerdeConfig.ENABLE_HEADERS, "true", SerdeConfig.USE_ID, IdOption.globalId.name()), - false); + deserializer.configure(Map.of(KafkaSerdeConfig.ENABLE_HEADERS, "true", SerdeConfig.USE_ID, + IdOption.globalId.name()), false); Headers headers = new RecordHeaders(); byte[] bytes = serializer.serialize(artifactId, headers, person); - Assertions.assertNotNull(headers.lastHeader(SerdeHeaders.HEADER_VALUE_GLOBAL_ID)); - Header headerGlobalId = headers.lastHeader(SerdeHeaders.HEADER_VALUE_GLOBAL_ID); + Assertions.assertNotNull(headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_GLOBAL_ID)); + Header headerGlobalId = headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_GLOBAL_ID); long id = ByteBuffer.wrap(headerGlobalId.value()).getLong(); assertEquals(globalId.intValue(), Long.valueOf(id).intValue()); - Assertions.assertNotNull(headers.lastHeader(SerdeHeaders.HEADER_VALUE_MESSAGE_TYPE)); - Header headerMsgType = headers.lastHeader(SerdeHeaders.HEADER_VALUE_MESSAGE_TYPE); + Assertions.assertNotNull(headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_MESSAGE_TYPE)); + Header headerMsgType = headers.lastHeader(KafkaSerdeHeaders.HEADER_VALUE_MESSAGE_TYPE); assertEquals(person.getClass().getName(), IoUtil.toString(headerMsgType.value())); person = deserializer.deserialize(artifactId, headers, bytes); @@ -239,16 +241,19 @@ public void testJsonSchemaSerdeMagicByte() throws Exception { Person person = new Person("Ales", "Justin", 23); - try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); + config.put(SerdeConfig.VALIDATION_ENABLED, "true"); serializer.configure(config, false); - deserializer.configure(Collections.singletonMap(SerdeConfig.ENABLE_HEADERS, "true"), false); + deserializer.configure( + Map.of(KafkaSerdeConfig.ENABLE_HEADERS, "true", SerdeConfig.VALIDATION_ENABLED, "true"), + false); byte[] bytes = serializer.serialize(artifactId, person); @@ -335,17 +340,19 @@ public void testJsonSchemaSerdeWithReferences() throws Exception { CitizenIdentifier identifier = new CitizenIdentifier(123456789); Citizen citizen = new Citizen("Carles", "Arnal", 23, city, identifier, Collections.emptyList()); - try ( - JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient, true); - Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient, true)) { + try (JsonSchemaKafkaSerializer serializer = new JsonSchemaKafkaSerializer<>(restClient); + Deserializer deserializer = new JsonSchemaKafkaDeserializer<>(restClient)) { Map config = new HashMap<>(); config.put(SerdeConfig.EXPLICIT_ARTIFACT_GROUP_ID, groupId); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, SimpleTopicIdStrategy.class.getName()); - config.put(SerdeConfig.ENABLE_HEADERS, "true"); + config.put(SerdeConfig.VALIDATION_ENABLED, "true"); + config.put(KafkaSerdeConfig.ENABLE_HEADERS, "true"); serializer.configure(config, false); - deserializer.configure(Collections.singletonMap(SerdeConfig.ENABLE_HEADERS, "true"), false); + deserializer.configure( + Map.of(KafkaSerdeConfig.ENABLE_HEADERS, "true", SerdeConfig.VALIDATION_ENABLED, "true"), + false); Headers headers = new RecordHeaders(); byte[] bytes = serializer.serialize(artifactId, headers, citizen); diff --git a/app/src/test/java/io/apicurio/registry/noprofile/serde/ProtobufSerdeTest.java b/app/src/test/java/io/apicurio/registry/noprofile/serde/ProtobufSerdeTest.java index 1b4f940d34..56566a2e73 100644 --- a/app/src/test/java/io/apicurio/registry/noprofile/serde/ProtobufSerdeTest.java +++ b/app/src/test/java/io/apicurio/registry/noprofile/serde/ProtobufSerdeTest.java @@ -8,7 +8,7 @@ import io.apicurio.registry.resolver.SchemaResolverConfig; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.rest.client.models.VersionMetaData; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.protobuf.ProtobufKafkaDeserializer; import io.apicurio.registry.serde.protobuf.ProtobufKafkaSerializer; import io.apicurio.registry.serde.strategy.SimpleTopicIdStrategy; diff --git a/docs/modules/ROOT/partials/getting-started/proc-migrating-registry-applications.adoc b/docs/modules/ROOT/partials/getting-started/proc-migrating-registry-applications.adoc index bf4f5ebe47..e502804d3e 100644 --- a/docs/modules/ROOT/partials/getting-started/proc-migrating-registry-applications.adoc +++ b/docs/modules/ROOT/partials/getting-started/proc-migrating-registry-applications.adoc @@ -5,36 +5,35 @@ = Migrating {registry} client applications [role="_abstract"] -You must review your existing {registry} client applications to ensure that the Maven dependencies and Java client configuration meet the new requirements for version 2.x. For example, this includes new Maven dependencies for the {registry} Java REST client libraries or Kafka client serializer/deserializer (Serdes) libraries. You must also update your Java application configuration with the new registry v2 API path. +You must review your existing {registry} client applications to ensure that the Maven dependencies and Java client configuration meet the new requirements for version 3.x. For example, this includes new Maven dependencies for the {registry} Java REST client libraries or Kafka client serializer/deserializer (Serdes) libraries. You must also update your Java application configuration with the new registry v3 API path. .Prerequisites -* Existing {registry} {registry-v1} Java client application or Kafka client producer and consumer Java applications with SerDes +* Existing {registry} {registry-v2} Java client application or Kafka client producer and consumer Java applications with SerDes .Procedure -. If you are using the {registry} Java REST client, you must change the Maven dependencies for the {registry} Java client libraries, which have been repackaged in version 2.x: +. If you are using the {registry} Java REST client, you must change the Maven dependencies for the {registry} Java client libraries, which have been repackaged in version 3.x: + [source, xml, subs="attributes+"] ---- io.apicurio - apicurio-registry-client + apicurio-registry-java-sdk {registry-release} ---- -. In your Java client application, you must change your registry URL configuration, from pointing to the existing v1 API path to the new v2 path. For example: +. In your Java client application, you must change your registry URL configuration, from pointing to the existing v2 API path to the new v3 path. Starting with v3, we use Vertx as the default platform for our rest client, due to this change, a RequestAdapter is required when the client is create. For example: + [source,java, subs="attributes+"] ---- public class ClientExample { - private static final RegistryRestClient client; public static void main(String[] args) throws Exception { - // Create a registry client - String registryUrl = "https://new-registry.my-company.com/apis/registry/v2"; - RegistryClient client = RegistryClientFactory.create(registryUrl); + VertXRequestAdapter vertXRequestAdapter = new VertXRequestAdapter(vertx); + vertXRequestAdapter.setBaseUrl("https://new-registry.my-company.com/apis/registry/v3"); + RegistryClient client = new RegistryClient(vertXRequestAdapter); } } ---- @@ -47,52 +46,44 @@ ifdef::rh-service-registry[] the link:{LinkServiceRegistryUser}#using-the-registry-client[{NameServiceRegistryUser}]. endif::[] -. If you are using the {registry} SerDes libraries, you must change the Maven dependencies, which have been repackaged in version 2.x. In {registry} {registry-v1}, the SerDes libraries were all provided with only one Maven dependency: +. If you are using the {registry} SerDes libraries, you must change the Maven dependencies, which have been repackaged in version 3.x. In {registry} {registry-v2}, the SerDes libraries were provided into three separate Maven modules, while this is still true, the names have been changed: + [source, xml, subs="attributes+"] ---- io.apicurio - apicurio-registry-utils-serde - {registry-v1-release} - ----- -+ -In {registry} 2.x, the SerDes libraries have been split into three Maven dependencies, one for each supported data format: `avro`, `protobuf`, and `json schema`, depending on your use cases: -+ -[source, xml, subs="attributes+"] ----- - - io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka {registry-release} io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-jsonschema-serde-kafka {registry-release} io.apicurio - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-protobuf-serde-kafka {registry-release} ---- ++ +In {registry} 3.x, the SerDes libraries have been significantly refactored to make them re-usable for other messaging platforms like Apache Pulsar, that's why the Apache Kafka specific ones have been renamed. ++ -. In your Kafka producer and consumer Java applications, you must change your registry URL configuration from pointing to the existing v1 API path to the new v2 path. For example: +. In your Kafka producer and consumer Java applications, you must change your registry URL configuration from pointing to the existing v2 API path to the new v3 path. For example: + -_Existing registry v1 API path_: +_Existing registry v2 API path_: + [source,java] ---- -props.putIfAbsent(AbstractKafkaSerDe.REGISTRY_URL_CONFIG_PARAM, "http://old-registry.my-company.com/api"); +props.putIfAbsent(SerdeConfig.REGISTRY_URL, "http://new-registry.my-company.com/apis/registry/v2"); ---- + -_New registry v2 API path_: +_New registry v3 API path_: + [source,java, subs="attributes+"] ---- -props.putIfAbsent(SerdeConfig.REGISTRY_URL, "http://new-registry.my-company.com/apis/registry/v2"); +props.putIfAbsent(SerdeConfig.REGISTRY_URL, "http://new-registry.my-company.com/apis/registry/v3"); ---- + The refactored SerDes libraries also include other important changes to configuration properties. For more details on SerDes configuration, see diff --git a/docs/modules/ROOT/partials/shared/attributes.adoc b/docs/modules/ROOT/partials/shared/attributes.adoc index ce7fc12ec0..cfb35c01e0 100644 --- a/docs/modules/ROOT/partials/shared/attributes.adoc +++ b/docs/modules/ROOT/partials/shared/attributes.adoc @@ -25,7 +25,8 @@ ifndef::service-registry-downstream[] :registry-release: 3.0.0 :registry-docker-version: latest-release :registry-v1: 1.3 -:registry-v1-release: 1.3.2.Final +:registry-v1-release: 1.3.2.Final +:registry-v2: 2.6.3 :operator-version: 1.1.0-v2.4.12.final :kafka-streams: Strimzi :registry-kafka-version: 3.5 diff --git a/examples/avro-bean/pom.xml b/examples/avro-bean/pom.xml index 606aac34f0..800d39df62 100644 --- a/examples/avro-bean/pom.xml +++ b/examples/avro-bean/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} diff --git a/examples/avro-bean/src/main/java/io/apicurio/registry/examples/avro/bean/AvroBeanExample.java b/examples/avro-bean/src/main/java/io/apicurio/registry/examples/avro/bean/AvroBeanExample.java index ad5abb2a7c..2537a1861b 100644 --- a/examples/avro-bean/src/main/java/io/apicurio/registry/examples/avro/bean/AvroBeanExample.java +++ b/examples/avro-bean/src/main/java/io/apicurio/registry/examples/avro/bean/AvroBeanExample.java @@ -16,11 +16,11 @@ package io.apicurio.registry.examples.avro.bean; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.avro.AvroSerdeConfig; import io.apicurio.registry.serde.avro.ReflectAvroDatumProvider; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -57,7 +57,7 @@ */ public class AvroBeanExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = AvroBeanExample.class.getSimpleName(); private static final String SUBJECT_NAME = "Greeting"; @@ -143,7 +143,7 @@ private static Producer createKafkaProducer() { props.putIfAbsent(SerdeConfig.AUTO_REGISTER_ARTIFACT, Boolean.TRUE); // Use Java reflection as the Avro Datum Provider - this also generates an Avro schema from the java // bean - props.putIfAbsent(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); + props.putIfAbsent(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); // Just if security values are present, then we configure them. configureSecurityIfPresent(props); @@ -174,7 +174,7 @@ private static KafkaConsumer createKafkaConsumer() { // Configure Service Registry location props.putIfAbsent(SerdeConfig.REGISTRY_URL, REGISTRY_URL); // Use Java reflection as the Avro Datum Provider - props.putIfAbsent(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); + props.putIfAbsent(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); // No other configuration needed for the deserializer, because the globalId of the schema // the deserializer should use is sent as part of the payload. So the deserializer simply // extracts that globalId and uses it to look up the Schema from the registry. diff --git a/examples/confluent-serdes/pom.xml b/examples/confluent-serdes/pom.xml index c26c711b05..d1166ad989 100644 --- a/examples/confluent-serdes/pom.xml +++ b/examples/confluent-serdes/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} diff --git a/examples/confluent-serdes/src/main/java/io/apicurio/registry/examples/confluent/serdes/ConfluentSerdesExample.java b/examples/confluent-serdes/src/main/java/io/apicurio/registry/examples/confluent/serdes/ConfluentSerdesExample.java index ca409f8eed..bc41814cf3 100644 --- a/examples/confluent-serdes/src/main/java/io/apicurio/registry/examples/confluent/serdes/ConfluentSerdesExample.java +++ b/examples/confluent-serdes/src/main/java/io/apicurio/registry/examples/confluent/serdes/ConfluentSerdesExample.java @@ -16,8 +16,8 @@ package io.apicurio.registry.examples.confluent.serdes; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; +import io.apicurio.registry.serde.config.SerdeConfig; import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.rest.RestService; import io.confluent.kafka.serializers.KafkaAvroSerializer; @@ -62,8 +62,8 @@ */ public class ConfluentSerdesExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; - private static final String CCOMPAT_API_URL = "http://localhost:8080/apis/ccompat/v6"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; + private static final String CCOMPAT_API_URL = "http://localhost:8080/apis/ccompat/v7"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = ConfluentSerdesExample.class.getSimpleName(); diff --git a/examples/custom-resolver/pom.xml b/examples/custom-resolver/pom.xml index 4ca9f86ba2..6c31737b1e 100644 --- a/examples/custom-resolver/pom.xml +++ b/examples/custom-resolver/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} diff --git a/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/Config.java b/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/Config.java index c5d9323d19..95d1d9d1db 100644 --- a/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/Config.java +++ b/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/Config.java @@ -21,7 +21,7 @@ */ public class Config { - public static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + public static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; public static final String SERVERS = "localhost:9092"; public static final String TOPIC_NAME = CustomSchemaResolverExample.class.getSimpleName(); public static final String SUBJECT_NAME = "Greeting"; diff --git a/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolver.java b/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolver.java index 32c9eda378..2c0add8076 100644 --- a/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolver.java +++ b/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolver.java @@ -16,24 +16,23 @@ package io.apicurio.registry.examples.custom.resolver; +import io.apicurio.registry.resolver.AbstractSchemaResolver; +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaLookupResult; +import io.apicurio.registry.resolver.data.Record; import io.apicurio.registry.rest.client.models.CreateArtifact; import io.apicurio.registry.rest.client.models.CreateVersion; import io.apicurio.registry.rest.client.models.IfArtifactExists; import io.apicurio.registry.rest.client.models.VersionContent; -import io.apicurio.registry.serde.AbstractSchemaResolver; -import io.apicurio.registry.serde.ParsedSchema; -import io.apicurio.registry.serde.SchemaLookupResult; -import io.apicurio.registry.serde.SchemaParser; -import io.apicurio.registry.serde.avro.AvroSchemaUtils; -import io.apicurio.registry.serde.strategy.ArtifactReference; +import io.apicurio.registry.rest.client.models.VersionMetaData; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.utils.IoUtil; import org.apache.avro.Schema; -import org.apache.kafka.common.header.Headers; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; /** @@ -47,30 +46,19 @@ public class CustomSchemaResolver extends AbstractSchemaResolver { protected final Map> schemaLookupCacheByContent = new ConcurrentHashMap<>(); - /** - * @see io.apicurio.registry.serde.SchemaResolver#configure(java.util.Map, boolean, - * io.apicurio.registry.serde.SchemaParser) - */ @Override - public void configure(Map configs, boolean isKey, SchemaParser schemaMapper) { - super.configure(configs, isKey, schemaMapper); + public SchemaLookupResult resolveSchemaByGlobalId(long globalId) { + return super.resolveSchemaByGlobalId(globalId); } - /** - * @see io.apicurio.registry.serde.SchemaResolver#resolveSchema(java.lang.String, - * org.apache.kafka.common.header.Headers, java.lang.Object, io.apicurio.registry.serde.ParsedSchema) - */ - @SuppressWarnings({ "unchecked", "rawtypes" }) @Override - public SchemaLookupResult resolveSchema(String topic, Headers headers, D data, - ParsedSchema parsedSchema) { - System.out.println("[CustomSchemaResolver] Resolving a schema for topic: " + topic); + public SchemaLookupResult resolveSchema(Record data) { + System.out.println("[CustomSchemaResolver] Resolving a schema"); String schema = Config.SCHEMA; return schemaLookupCacheByContent.computeIfAbsent(schema, (schemaData) -> { String groupId = "default"; - String artifactId = topic + "-value"; - Schema schemaObj = AvroSchemaUtils.parse(schema); + String artifactId = UUID.randomUUID() + "-value"; ByteArrayInputStream schemaContent = new ByteArrayInputStream( schema.getBytes(StandardCharsets.UTF_8)); @@ -84,33 +72,30 @@ public SchemaLookupResult resolveSchema(String topic, Headers headers, D createArtifact.getFirstVersion().getContent().setContent(IoUtil.toString(schemaContent)); createArtifact.getFirstVersion().getContent().setContentType("application/json"); - final io.apicurio.registry.rest.client.models.VersionMetaData metaData = client.groups() - .byGroupId("default").artifacts().post(createArtifact, config -> { + final VersionMetaData metaData = client.groups().byGroupId("default").artifacts() + .post(createArtifact, config -> { config.queryParameters.ifExists = IfArtifactExists.FIND_OR_CREATE_VERSION; }).getVersion(); - SchemaLookupResult result = SchemaLookupResult.builder().groupId(groupId).artifactId(artifactId) - .version(String.valueOf(metaData.getVersion())).globalId(metaData.getGlobalId()) - .schema(schemaObj).rawSchema(schema.getBytes(StandardCharsets.UTF_8)).build(); + ParsedSchema parsedSchema = this.schemaParser.getSchemaFromData(data); + + SchemaLookupResult.SchemaLookupResultBuilder lookupResultBuilder = SchemaLookupResult + .builder(); - // Also update the schemaCacheByGlobalId - useful if this resolver is used by both + SchemaLookupResult result = lookupResultBuilder.groupId(groupId).artifactId(artifactId) + .version(String.valueOf(metaData.getVersion())).contentId(metaData.getContentId()) + .parsedSchema(parsedSchema).build(); + + // Also update the schemaCacheByContentId - useful if this resolver is used by both // the serializer and deserializer in the same Java application. - return schemaCache.getByGlobalId(metaData.getGlobalId(), (id) -> result); + return schemaCache.getByContentId(metaData.getContentId(), (id) -> result); }); } - /** - * @see io.apicurio.registry.serde.SchemaResolver#resolveSchemaByArtifactReference(io.apicurio.registry.serde.strategy.ArtifactReference) - */ @Override - public SchemaLookupResult resolveSchemaByArtifactReference(ArtifactReference reference) { + public SchemaLookupResult resolveSchemaByArtifactReference( + io.apicurio.registry.resolver.strategy.ArtifactReference reference) { throw new UnsupportedOperationException( "resolveSchemaByArtifactReference() is not supported by this implementation."); } - - @Override - public SchemaLookupResult resolveSchemaByGlobalId(long globalId) { - return super.resolveSchemaByGlobalId(globalId); - } - } diff --git a/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolverExample.java b/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolverExample.java index 69c0228e6c..24ab0be6df 100644 --- a/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolverExample.java +++ b/examples/custom-resolver/src/main/java/io/apicurio/registry/examples/custom/resolver/CustomSchemaResolverExample.java @@ -16,9 +16,9 @@ package io.apicurio.registry.examples.custom.resolver; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -170,9 +170,9 @@ private static KafkaConsumer createKafkaConsumer() { // Configure Service Registry location props.putIfAbsent(SerdeConfig.REGISTRY_URL, Config.REGISTRY_URL); - // No other configuration needed for the deserializer, because the globalId of the schema + // No other configuration needed for the deserializer, because the contentId of the schema // the deserializer should use is sent as part of the payload. So the deserializer simply - // extracts that globalId and uses it to look up the Schema from the registry. + // extracts that contentId and uses it to look up the Schema from the registry. // Just if security values are present, then we configure them. configureSecurityIfPresent(props); diff --git a/examples/custom-strategy/pom.xml b/examples/custom-strategy/pom.xml index bab392953a..58a65280ad 100644 --- a/examples/custom-strategy/pom.xml +++ b/examples/custom-strategy/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} diff --git a/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/Config.java b/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/Config.java index ad27ba00c0..00fa5ab96e 100644 --- a/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/Config.java +++ b/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/Config.java @@ -21,7 +21,7 @@ */ public class Config { - public static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + public static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; public static final String SERVERS = "localhost:9092"; public static final String TOPIC_NAME = CustomStrategyExample.class.getSimpleName(); public static final String SUBJECT_NAME = "Greeting"; diff --git a/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/CustomStrategyExample.java b/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/CustomStrategyExample.java index 297a4830ed..767a834825 100644 --- a/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/CustomStrategyExample.java +++ b/examples/custom-strategy/src/main/java/io/apicurio/registry/examples/custom/strategy/CustomStrategyExample.java @@ -22,9 +22,9 @@ import io.apicurio.registry.rest.client.models.CreateVersion; import io.apicurio.registry.rest.client.models.IfArtifactExists; import io.apicurio.registry.rest.client.models.VersionContent; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.types.ArtifactType; import io.kiota.http.vertx.VertXRequestAdapter; import org.apache.avro.Schema; diff --git a/examples/debezium-openshift/pom.xml b/examples/debezium-openshift/pom.xml index 7b2b75029f..877b29d9dd 100644 --- a/examples/debezium-openshift/pom.xml +++ b/examples/debezium-openshift/pom.xml @@ -56,7 +56,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} diff --git a/examples/debezium-openshift/src/main/java/io/apicurio/example/debezium/kafka/KafkaFactory.java b/examples/debezium-openshift/src/main/java/io/apicurio/example/debezium/kafka/KafkaFactory.java index b619f1c979..67829def96 100644 --- a/examples/debezium-openshift/src/main/java/io/apicurio/example/debezium/kafka/KafkaFactory.java +++ b/examples/debezium-openshift/src/main/java/io/apicurio/example/debezium/kafka/KafkaFactory.java @@ -1,8 +1,8 @@ package io.apicurio.example.debezium.kafka; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; +import io.apicurio.registry.serde.avro.AvroSerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.eclipse.microprofile.config.inject.ConfigProperty; @@ -44,7 +44,7 @@ public KafkaConsumer createKafkaConsumer() { log.debug("Registry URL: {}", registryUrl); props.putIfAbsent(SerdeConfig.REGISTRY_URL, registryUrl); // Deserialize into a specific class instead of GenericRecord - props.putIfAbsent(AvroKafkaSerdeConfig.USE_SPECIFIC_AVRO_READER, true); + props.putIfAbsent(AvroSerdeConfig.USE_SPECIFIC_AVRO_READER, true); return new KafkaConsumer<>(props); } diff --git a/examples/developer-basic-auth/basic-auth.properties b/examples/developer-basic-auth/basic-auth.properties index 09bd1fde26..f5eb8e175a 100644 --- a/examples/developer-basic-auth/basic-auth.properties +++ b/examples/developer-basic-auth/basic-auth.properties @@ -14,7 +14,7 @@ quarkus.security.users.embedded.plain-text=true quarkus.security.users.embedded.users.admin=admin quarkus.security.users.embedded.roles.admin=sr-admin # example: -# curl -H "Authorization: Basic $(echo -n "admin:admin" | base64)" http://localhost:8080/apis/registry/v2/admin/rules -v +# curl -H "Authorization: Basic $(echo -n "admin:admin" | base64)" http://localhost:8080/apis/registry/v3/admin/rules -v quarkus.security.users.embedded.users.developer=developer quarkus.security.users.embedded.roles.developer=sr-developer quarkus.security.users.embedded.users.readonly=readonly diff --git a/examples/jsonschema-validation/src/main/java/io/apicurio/registry/examples/validation/json/JsonSchemaValidationExample.java b/examples/jsonschema-validation/src/main/java/io/apicurio/registry/examples/validation/json/JsonSchemaValidationExample.java index f99ce4cd14..708f422aab 100644 --- a/examples/jsonschema-validation/src/main/java/io/apicurio/registry/examples/validation/json/JsonSchemaValidationExample.java +++ b/examples/jsonschema-validation/src/main/java/io/apicurio/registry/examples/validation/json/JsonSchemaValidationExample.java @@ -63,7 +63,7 @@ */ public class JsonSchemaValidationExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; public static final String SCHEMA = "{" + " \"$id\": \"https://example.com/message.schema.json\"," + " \"$schema\": \"http://json-schema.org/draft-07/schema#\"," + " \"required\": [" diff --git a/examples/mix-avro/pom.xml b/examples/mix-avro/pom.xml index 9fb3f9d2b9..4c8fa4bb19 100644 --- a/examples/mix-avro/pom.xml +++ b/examples/mix-avro/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} diff --git a/examples/mix-avro/src/main/java/io/apicurio/registry/examples/mix/avro/MixAvroExample.java b/examples/mix-avro/src/main/java/io/apicurio/registry/examples/mix/avro/MixAvroExample.java index c1346455f6..a18d0ddfc9 100644 --- a/examples/mix-avro/src/main/java/io/apicurio/registry/examples/mix/avro/MixAvroExample.java +++ b/examples/mix-avro/src/main/java/io/apicurio/registry/examples/mix/avro/MixAvroExample.java @@ -16,10 +16,10 @@ package io.apicurio.registry.examples.mix.avro; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; import io.apicurio.registry.serde.avro.strategy.RecordIdStrategy; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -65,7 +65,7 @@ */ public class MixAvroExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = MixAvroExample.class.getSimpleName(); private static final String SCHEMAV1 = "{\"type\":\"record\",\"name\":\"Greeting\",\"fields\":[{\"name\":\"Message\",\"type\":\"string\"},{\"name\":\"Time\",\"type\":\"long\"}]}"; diff --git a/examples/protobuf-bean/pom.xml b/examples/protobuf-bean/pom.xml index 8137eb7cce..81d5bdd21a 100644 --- a/examples/protobuf-bean/pom.xml +++ b/examples/protobuf-bean/pom.xml @@ -19,7 +19,7 @@ io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-protobuf-serde-kafka ${project.version} diff --git a/examples/protobuf-bean/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufBeanExample.java b/examples/protobuf-bean/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufBeanExample.java index acb98c4edc..0cf535ea27 100644 --- a/examples/protobuf-bean/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufBeanExample.java +++ b/examples/protobuf-bean/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufBeanExample.java @@ -21,7 +21,7 @@ import io.apicurio.registry.examples.AddressBookProtos.AddressBook; import io.apicurio.registry.examples.AddressBookProtos.Person; import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.protobuf.ProtobufKafkaDeserializer; import io.apicurio.registry.serde.protobuf.ProtobufKafkaSerializer; import io.apicurio.registry.utils.IoUtil; @@ -61,7 +61,7 @@ */ public class ProtobufBeanExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = ProtobufBeanExample.class.getSimpleName(); private static final String SCHEMA_NAME = "AddressBook"; diff --git a/examples/protobuf-find-latest/pom.xml b/examples/protobuf-find-latest/pom.xml index d9a3e1cb22..002855c63c 100644 --- a/examples/protobuf-find-latest/pom.xml +++ b/examples/protobuf-find-latest/pom.xml @@ -19,7 +19,7 @@ io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-protobuf-serde-kafka ${project.version} diff --git a/examples/protobuf-find-latest/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufFindLatestExample.java b/examples/protobuf-find-latest/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufFindLatestExample.java index b1ed3bb4a8..f497ba78c9 100644 --- a/examples/protobuf-find-latest/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufFindLatestExample.java +++ b/examples/protobuf-find-latest/src/main/java/io/apicurio/registry/examples/simple/protobuf/ProtobufFindLatestExample.java @@ -25,7 +25,7 @@ import io.apicurio.registry.rest.client.models.CreateVersion; import io.apicurio.registry.rest.client.models.IfArtifactExists; import io.apicurio.registry.rest.client.models.VersionContent; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.protobuf.ProtobufKafkaDeserializer; import io.apicurio.registry.serde.protobuf.ProtobufKafkaSerializer; import io.apicurio.registry.types.ArtifactType; @@ -71,7 +71,7 @@ */ public class ProtobufFindLatestExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = ProtobufFindLatestExample.class.getSimpleName(); private static final String SCHEMA_NAME = "AddressBook"; diff --git a/examples/protobuf-validation/src/main/java/io/apicurio/registry/examples/validation/protobuf/ProtobufValidationExample.java b/examples/protobuf-validation/src/main/java/io/apicurio/registry/examples/validation/protobuf/ProtobufValidationExample.java index 4736224e1e..00e50a7ebc 100644 --- a/examples/protobuf-validation/src/main/java/io/apicurio/registry/examples/validation/protobuf/ProtobufValidationExample.java +++ b/examples/protobuf-validation/src/main/java/io/apicurio/registry/examples/validation/protobuf/ProtobufValidationExample.java @@ -66,7 +66,7 @@ */ public class ProtobufValidationExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; public static final String SCHEMA = "syntax = \"proto3\";\n" + "package io.apicurio.schema.validation.protobuf.ref;\n" + "\n" + "message MessageExample {\n" diff --git a/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemo.java b/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemo.java index bf9c59e550..1f0509de64 100644 --- a/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemo.java +++ b/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemo.java @@ -23,7 +23,7 @@ public class SimpleRegistryDemo { static { // Create a Service Registry client - String registryUrl = "http://localhost:8080/apis/registry/v2"; + String registryUrl = "http://localhost:8080/apis/registry/v3"; vertx = Vertx.vertx(); client = createProperClient(registryUrl); } diff --git a/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemoBasicAuth.java b/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemoBasicAuth.java index ca0223f721..a0fdfccfe0 100644 --- a/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemoBasicAuth.java +++ b/examples/rest-client/src/main/java/io/apicurio/registry/examples/SimpleRegistryDemoBasicAuth.java @@ -22,7 +22,7 @@ public class SimpleRegistryDemoBasicAuth { static { // Create a Service Registry client - String registryUrl = "http://localhost:8080/apis/registry/v2"; + String registryUrl = "http://localhost:8080/apis/registry/v3"; client = createProperClient(registryUrl); } diff --git a/examples/serdes-with-references/pom.xml b/examples/serdes-with-references/pom.xml index 75e72152c9..558be742b4 100644 --- a/examples/serdes-with-references/pom.xml +++ b/examples/serdes-with-references/pom.xml @@ -20,17 +20,17 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} io.apicurio - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-jsonschema-serde-kafka ${project.version} io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-protobuf-serde-kafka ${project.version} diff --git a/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/AvroSerdeReferencesExample.java b/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/AvroSerdeReferencesExample.java index a6fdaebb04..5815145671 100644 --- a/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/AvroSerdeReferencesExample.java +++ b/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/AvroSerdeReferencesExample.java @@ -4,11 +4,11 @@ import com.kubetrade.schema.trade.TradeKey; import com.kubetrade.schema.trade.TradeRaw; import io.apicurio.registry.rest.v2.beans.IfExists; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.avro.AvroSerdeConfig; import io.apicurio.registry.serde.avro.ReflectAvroDatumProvider; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -26,7 +26,7 @@ public class AvroSerdeReferencesExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = AvroSerdeReferencesExample.class.getSimpleName(); private static final String SUBJECT_NAME = "Trade"; @@ -116,7 +116,7 @@ private static Producer createKafkaProducer() { // Register the artifact if not found in the registry. props.putIfAbsent(SerdeConfig.AUTO_REGISTER_ARTIFACT, Boolean.TRUE); props.putIfAbsent(SerdeConfig.AUTO_REGISTER_ARTIFACT_IF_EXISTS, IfExists.RETURN.name()); - props.put(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); + props.put(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); // Just if security values are present, then we configure them. configureSecurityIfPresent(props); @@ -142,8 +142,8 @@ private static KafkaConsumer createKafkaConsumer() { // Use the Apicurio Registry provided Kafka Deserializer for Avro props.putIfAbsent(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, AvroKafkaDeserializer.class.getName()); - props.putIfAbsent(AvroKafkaSerdeConfig.AVRO_ENCODING, AvroKafkaSerdeConfig.AVRO_ENCODING_JSON); - props.putIfAbsent(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); + props.putIfAbsent(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON); + props.putIfAbsent(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()); // Configure Service Registry location props.putIfAbsent(SerdeConfig.REGISTRY_URL, REGISTRY_URL); diff --git a/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/JsonSerdeReferencesExample.java b/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/JsonSerdeReferencesExample.java index b6ada3ad31..3dfede677d 100644 --- a/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/JsonSerdeReferencesExample.java +++ b/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/JsonSerdeReferencesExample.java @@ -10,7 +10,7 @@ import io.apicurio.registry.rest.client.models.CreateVersion; import io.apicurio.registry.rest.client.models.IfArtifactExists; import io.apicurio.registry.rest.client.models.VersionContent; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaDeserializer; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaSerializer; import io.apicurio.registry.serde.strategy.SimpleTopicIdStrategy; @@ -37,7 +37,7 @@ import static io.apicurio.registry.client.auth.VertXAuthFactory.buildOIDCWebClient; public class JsonSerdeReferencesExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = JsonSerdeReferencesExample.class.getSimpleName(); private static final String SUBJECT_NAME = "Greeting"; diff --git a/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/ProtobufSerdeReferencesExample.java b/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/ProtobufSerdeReferencesExample.java index 1ccae9c316..537f2f3749 100644 --- a/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/ProtobufSerdeReferencesExample.java +++ b/examples/serdes-with-references/src/main/java/io/apicurio/registry/examples/references/ProtobufSerdeReferencesExample.java @@ -6,7 +6,7 @@ import io.api.sample.TableNotification; import io.api.sample.TableNotificationType; import io.apicurio.registry.rest.v2.beans.IfExists; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.protobuf.ProtobufKafkaDeserializer; import io.apicurio.registry.serde.protobuf.ProtobufKafkaSerializer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -28,7 +28,7 @@ public class ProtobufSerdeReferencesExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = ProtobufSerdeReferencesExample.class.getSimpleName(); private static final String SUBJECT_NAME = "TableNotification"; diff --git a/examples/simple-avro-downstream/pom.xml b/examples/simple-avro-downstream/pom.xml index c29c91162d..ff28881af5 100644 --- a/examples/simple-avro-downstream/pom.xml +++ b/examples/simple-avro-downstream/pom.xml @@ -21,7 +21,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${apicurio-registry.version} diff --git a/examples/simple-avro-downstream/src/main/java/avro/SimpleAvroExample.java b/examples/simple-avro-downstream/src/main/java/avro/SimpleAvroExample.java index 08a085f36a..2a7baba7f3 100644 --- a/examples/simple-avro-downstream/src/main/java/avro/SimpleAvroExample.java +++ b/examples/simple-avro-downstream/src/main/java/avro/SimpleAvroExample.java @@ -16,9 +16,9 @@ package avro; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; diff --git a/examples/simple-avro-maven/pom.xml b/examples/simple-avro-maven/pom.xml index 985f492527..a1ba1d13a4 100644 --- a/examples/simple-avro-maven/pom.xml +++ b/examples/simple-avro-maven/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} @@ -57,7 +57,7 @@ register - http://localhost:8080/apis/registry/v2 + http://localhost:8080/apis/registry/v3 default diff --git a/examples/simple-avro-maven/src/main/java/io/apicurio/registry/examples/simple/avro/maven/SimpleAvroMavenExample.java b/examples/simple-avro-maven/src/main/java/io/apicurio/registry/examples/simple/avro/maven/SimpleAvroMavenExample.java index 490a4366ff..ca9be21d35 100644 --- a/examples/simple-avro-maven/src/main/java/io/apicurio/registry/examples/simple/avro/maven/SimpleAvroMavenExample.java +++ b/examples/simple-avro-maven/src/main/java/io/apicurio/registry/examples/simple/avro/maven/SimpleAvroMavenExample.java @@ -20,9 +20,9 @@ import io.apicurio.registry.client.auth.VertXAuthFactory; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.rest.v2.beans.IfExists; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; import io.kiota.http.vertx.VertXRequestAdapter; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; @@ -73,7 +73,7 @@ */ public class SimpleAvroMavenExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = SimpleAvroMavenExample.class.getSimpleName(); private static final String SUBJECT_NAME = "Greeting"; diff --git a/examples/simple-avro/pom.xml b/examples/simple-avro/pom.xml index c4a18314aa..9c8e0cd743 100644 --- a/examples/simple-avro/pom.xml +++ b/examples/simple-avro/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka ${project.version} diff --git a/examples/simple-avro/src/main/java/io/apicurio/registry/examples/simple/avro/SimpleAvroExample.java b/examples/simple-avro/src/main/java/io/apicurio/registry/examples/simple/avro/SimpleAvroExample.java index 0f87213a95..fa35cb3408 100644 --- a/examples/simple-avro/src/main/java/io/apicurio/registry/examples/simple/avro/SimpleAvroExample.java +++ b/examples/simple-avro/src/main/java/io/apicurio/registry/examples/simple/avro/SimpleAvroExample.java @@ -16,9 +16,9 @@ package io.apicurio.registry.examples.simple.avro; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -58,7 +58,7 @@ */ public class SimpleAvroExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = SimpleAvroExample.class.getSimpleName(); private static final String SUBJECT_NAME = "Greeting"; diff --git a/examples/simple-json/pom.xml b/examples/simple-json/pom.xml index 5d3c694c23..ee62c37ac0 100644 --- a/examples/simple-json/pom.xml +++ b/examples/simple-json/pom.xml @@ -18,7 +18,7 @@ io.apicurio - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-jsonschema-serde-kafka ${project.version} diff --git a/examples/simple-json/src/main/java/io/apicurio/registry/examples/simple/json/SimpleJsonSchemaExample.java b/examples/simple-json/src/main/java/io/apicurio/registry/examples/simple/json/SimpleJsonSchemaExample.java index 14e8ef73ff..b3c255824f 100644 --- a/examples/simple-json/src/main/java/io/apicurio/registry/examples/simple/json/SimpleJsonSchemaExample.java +++ b/examples/simple-json/src/main/java/io/apicurio/registry/examples/simple/json/SimpleJsonSchemaExample.java @@ -23,7 +23,7 @@ import io.apicurio.registry.rest.client.models.IfArtifactExists; import io.apicurio.registry.rest.client.models.VersionContent; import io.apicurio.registry.rest.client.models.VersionMetaData; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaDeserializer; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaSerializer; import io.apicurio.registry.types.ArtifactType; @@ -71,7 +71,7 @@ */ public class SimpleJsonSchemaExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = SimpleJsonSchemaExample.class.getSimpleName(); private static final String SUBJECT_NAME = "Greeting"; diff --git a/examples/simple-protobuf/pom.xml b/examples/simple-protobuf/pom.xml index 2be5eb2b4d..f0dcf98406 100644 --- a/examples/simple-protobuf/pom.xml +++ b/examples/simple-protobuf/pom.xml @@ -19,7 +19,7 @@ io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-protobuf-serde-kafka ${project.version} diff --git a/examples/simple-protobuf/src/main/java/io/apicurio/registry/examples/simple/protobuf/SimpleProtobufExample.java b/examples/simple-protobuf/src/main/java/io/apicurio/registry/examples/simple/protobuf/SimpleProtobufExample.java index 2664d7943f..3bfd1fc635 100644 --- a/examples/simple-protobuf/src/main/java/io/apicurio/registry/examples/simple/protobuf/SimpleProtobufExample.java +++ b/examples/simple-protobuf/src/main/java/io/apicurio/registry/examples/simple/protobuf/SimpleProtobufExample.java @@ -22,7 +22,7 @@ import io.apicurio.registry.examples.AddressBookProtos.AddressBook; import io.apicurio.registry.examples.AddressBookProtos.Person; import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.protobuf.ProtobufKafkaDeserializer; import io.apicurio.registry.serde.protobuf.ProtobufKafkaSerializer; import io.apicurio.registry.utils.IoUtil; @@ -62,7 +62,7 @@ */ public class SimpleProtobufExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String SERVERS = "localhost:9092"; private static final String TOPIC_NAME = SimpleProtobufExample.class.getSimpleName(); private static final String SCHEMA_NAME = "AddressBook"; diff --git a/examples/simple-validation/src/main/java/io/apicurio/registry/examples/simple/json/SimpleValidationExample.java b/examples/simple-validation/src/main/java/io/apicurio/registry/examples/simple/json/SimpleValidationExample.java index 51f2900fa8..bc5954fde6 100644 --- a/examples/simple-validation/src/main/java/io/apicurio/registry/examples/simple/json/SimpleValidationExample.java +++ b/examples/simple-validation/src/main/java/io/apicurio/registry/examples/simple/json/SimpleValidationExample.java @@ -41,7 +41,7 @@ */ public class SimpleValidationExample { - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; + private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v3"; private static final String GROUP = "Examples"; private static final String ARTIFACT_ID = "MessageType"; private static final SecureRandom rand = new SecureRandom(); diff --git a/examples/simple-validation/src/main/scripts/consumer.sh b/examples/simple-validation/src/main/scripts/consumer.sh index 6ad934de9c..5f60f47257 100644 --- a/examples/simple-validation/src/main/scripts/consumer.sh +++ b/examples/simple-validation/src/main/scripts/consumer.sh @@ -2,7 +2,7 @@ DONE=false -curl -s http://localhost:8080/apis/registry/v2/groups/Examples/artifacts/MessageType > _schema.json +curl -s http://localhost:8080/apis/registry/v3/groups/Examples/artifacts/MessageType > _schema.json echo "Schema fetched from registry." cat _schema.json diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index a5e78b5e79..dc0d8879a5 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -76,17 +76,17 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-jsonschema-serde-kafka test io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-avro-serde-kafka test io.apicurio - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-protobuf-serde-kafka test diff --git a/integration-tests/src/test/java/io/apicurio/deployment/RegistryDeploymentManager.java b/integration-tests/src/test/java/io/apicurio/deployment/RegistryDeploymentManager.java index 106a11cc91..f1d9ce09ce 100644 --- a/integration-tests/src/test/java/io/apicurio/deployment/RegistryDeploymentManager.java +++ b/integration-tests/src/test/java/io/apicurio/deployment/RegistryDeploymentManager.java @@ -40,7 +40,9 @@ public class RegistryDeploymentManager implements TestExecutionListener { @Override public void executionStarted(TestIdentifier testIdentifier) { TestExecutionListener.super.executionStarted(testIdentifier); - logWatch = streamPodLogs(testLogsIdentifier); + if (Boolean.parseBoolean(System.getProperty("cluster.tests"))) { + logWatch = streamPodLogs(testLogsIdentifier); + } } @Override diff --git a/integration-tests/src/test/java/io/apicurio/tests/converters/RegistryConverterIT.java b/integration-tests/src/test/java/io/apicurio/tests/converters/RegistryConverterIT.java index 3b4ec7164e..5e390b9181 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/converters/RegistryConverterIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/converters/RegistryConverterIT.java @@ -4,12 +4,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.serde.AbstractKafkaSerDe; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.avro.AvroSerdeConfig; import io.apicurio.registry.serde.avro.DefaultAvroDatumProvider; import io.apicurio.registry.serde.avro.strategy.TopicRecordIdStrategy; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.utils.converter.AvroConverter; @@ -67,17 +67,14 @@ public void testConfiguration() throws Exception { config.put(SerdeBasedConverter.REGISTRY_CONVERTER_DESERIALIZER_PARAM, AvroKafkaDeserializer.class.getName()); config.put(SerdeConfig.ARTIFACT_RESOLVER_STRATEGY, TopicRecordIdStrategy.class.getName()); - config.put(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class.getName()); - SerdeBasedConverter converter = new SerdeBasedConverter<>(); + config.put(AvroSerdeConfig.AVRO_DATUM_PROVIDER, DefaultAvroDatumProvider.class.getName()); - byte[] bytes; - try { + try (SerdeBasedConverter converter = new SerdeBasedConverter<>()) { + byte[] bytes; converter.configure(config, true); bytes = converter.fromConnectData(topic, null, record); record = (Record) converter.toConnectData(topic, bytes).value(); Assertions.assertEquals("somebar", record.get("bar").toString()); - } finally { - converter.close(); } } diff --git a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/AvroSerdeIT.java b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/AvroSerdeIT.java index 074e097b6c..39aac8829e 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/AvroSerdeIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/AvroSerdeIT.java @@ -1,14 +1,14 @@ package io.apicurio.tests.serdes.apicurio; import io.apicurio.registry.rest.client.models.VersionMetaData; -import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.apicurio.registry.serde.avro.AvroKafkaSerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaSerializer; +import io.apicurio.registry.serde.avro.AvroSerdeConfig; import io.apicurio.registry.serde.avro.ReflectAvroDatumProvider; import io.apicurio.registry.serde.avro.strategy.RecordIdStrategy; import io.apicurio.registry.serde.avro.strategy.TopicRecordIdStrategy; import io.apicurio.registry.serde.config.IdOption; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.strategy.SimpleTopicIdStrategy; import io.apicurio.registry.serde.strategy.TopicIdStrategy; import io.apicurio.registry.types.ArtifactType; @@ -499,10 +499,8 @@ public void testAvroJSON() throws Exception { .withSerializer(serializer).withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withDataGenerator(avroSchema::generateRecord).withDataValidator(avroSchema::validateRecord) .withProducerProperty(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true") - .withProducerProperty(AvroKafkaSerdeConfig.AVRO_ENCODING, - AvroKafkaSerdeConfig.AVRO_ENCODING_JSON) - .withConsumerProperty(AvroKafkaSerdeConfig.AVRO_ENCODING, - AvroKafkaSerdeConfig.AVRO_ENCODING_JSON) + .withProducerProperty(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON) + .withConsumerProperty(AvroSerdeConfig.AVRO_ENCODING, AvroSerdeConfig.AVRO_ENCODING_JSON) .withAfterProduceValidator(() -> { return TestUtils.retry(() -> { VersionMetaData meta = registryClient.groups().byGroupId("default").artifacts() @@ -528,10 +526,10 @@ public void testReflectAutoRegister() throws Exception { .withStrategy(TopicIdStrategy.class).withSerializer(serializer).withDeserializer(deserializer) .withDataGenerator(i -> new TestObject("Apicurio")) .withDataValidator(o -> "Apicurio".equals(o.getName())) - .withProducerProperty(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, + .withProducerProperty(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()) .withProducerProperty(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true") - .withConsumerProperty(AvroKafkaSerdeConfig.AVRO_DATUM_PROVIDER, + .withConsumerProperty(AvroSerdeConfig.AVRO_DATUM_PROVIDER, ReflectAvroDatumProvider.class.getName()) .withAfterProduceValidator(() -> { return TestUtils.retry(() -> { diff --git a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/JsonSchemaSerdeIT.java b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/JsonSchemaSerdeIT.java index 01c6c3e091..7046f769c0 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/JsonSchemaSerdeIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/JsonSchemaSerdeIT.java @@ -1,7 +1,8 @@ package io.apicurio.tests.serdes.apicurio; import com.fasterxml.jackson.databind.JsonNode; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.KafkaSerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaDeserializer; import io.apicurio.registry.serde.jsonschema.JsonSchemaKafkaSerializer; import io.apicurio.registry.serde.strategy.SimpleTopicIdStrategy; @@ -54,7 +55,7 @@ void testTopicIdStrategyFindLatest() throws Exception { ContentTypes.APPLICATION_JSON, null, null); new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withDataGenerator(schema::generateMessage).withDataValidator(schema::validateMessage).build() .test(); @@ -72,7 +73,7 @@ void testSimpleTopicIdStrategyFindLatest() throws Exception { ContentTypes.APPLICATION_JSON, null, null); new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withDeserializer(deserializer).withStrategy(SimpleTopicIdStrategy.class) .withDataGenerator(schema::generateMessage).withDataValidator(schema::validateMessage).build() .test(); diff --git a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/ProtobufSerdeIT.java b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/ProtobufSerdeIT.java index d6560808b4..4b94f29c9f 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/ProtobufSerdeIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/ProtobufSerdeIT.java @@ -3,9 +3,10 @@ import com.google.protobuf.DynamicMessage; import io.apicurio.registry.rest.client.models.ProblemDetails; import io.apicurio.registry.rest.client.models.VersionMetaData; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.KafkaSerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; +import io.apicurio.registry.serde.protobuf.ProtobufDeserializerConfig; import io.apicurio.registry.serde.protobuf.ProtobufKafkaDeserializer; -import io.apicurio.registry.serde.protobuf.ProtobufKafkaDeserializerConfig; import io.apicurio.registry.serde.protobuf.ProtobufKafkaSerializer; import io.apicurio.registry.serde.strategy.SimpleTopicIdStrategy; import io.apicurio.registry.serde.strategy.TopicIdStrategy; @@ -57,7 +58,7 @@ void testTopicIdStrategyFindLatest() throws Exception { logRestClientError(() -> { new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withDataGenerator(schema::generateMessage).withDataValidator(schema::validateMessage) .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "true").build(); @@ -77,7 +78,7 @@ void testSimpleTopicIdStrategyFindLatest() throws Exception { logRestClientError(() -> { new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withDeserializer(deserializer).withStrategy(SimpleTopicIdStrategy.class) .withDataGenerator(schema::generateMessage).withDataValidator(schema::validateMessage) .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "true").build(); @@ -168,21 +169,21 @@ void testValidation() throws Exception { // by default the artifact is found by content so this should work by finding the version 1 of the // artifact new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "false") .withProducerProperty(SerdeConfig.EXPLICIT_ARTIFACT_VERSION, "1") .withDataGenerator(schemaV1::generateMessage).withDataValidator(schemaV1::validateMessage) .build().test(); new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "false") .withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withDataGenerator(schemaV1::generateMessage).withDataValidator(schemaV1::validateMessage) .build().test(); new SimpleSerdesTesterBuilder().withTopic(topicName) .withSerializer(serializer).withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true") + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true") .withDataGenerator(schemaV2::generateMessage).withDataValidator(schemaV2::validateTypeMessage) .build().test(); @@ -196,13 +197,13 @@ void testValidation() throws Exception { // if find latest is enabled and we use the v2 schema it should work. Validation is enabled by default new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "true") .withDataGenerator(schemaV2::generateMessage).withDataValidator(schemaV2::validateTypeMessage) .build().test(); new SimpleSerdesTesterBuilder().withTopic(topicName) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true").withSerializer(serializer) .withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withProducerProperty(SerdeConfig.EXPLICIT_ARTIFACT_VERSION, "2") .withDataGenerator(schemaV2::generateMessage).withDataValidator(schemaV2::validateTypeMessage) @@ -269,7 +270,7 @@ void testFindLatestDeriveClassProtobufTypeTopicIdStrategy() throws Exception { .withSerializer(serializer).withDeserializer(deserializer) .withStrategy(TopicIdStrategy.class).withDataGenerator(schema::generateMessage) .withDataValidator(schema::validateMessage) - .withConsumerProperty(ProtobufKafkaDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") + .withConsumerProperty(ProtobufDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "true").build(); }); } @@ -290,7 +291,7 @@ public void testFindLatestDeriveClassProtobufTypeSimpleTopicIdStrategy() throws .withSerializer(serializer).withDeserializer(deserializer) .withStrategy(SimpleTopicIdStrategy.class).withDataGenerator(schema::generateMessage) .withDataValidator(schema::validateMessage) - .withConsumerProperty(ProtobufKafkaDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") + .withConsumerProperty(ProtobufDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "true").build(); }); } @@ -312,7 +313,7 @@ public void testFindLatestSpecificProtobufType() throws Exception { logRestClientError(() -> { new SimpleSerdesTesterBuilder().withTopic(topicName) .withSerializer(serializer).withDeserializer(deserializer) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true") + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true") .withStrategy(SimpleTopicIdStrategy.class).withDataGenerator(schema::generateMessage) .withDataValidator(schema::validateMessage) .withProducerProperty(SerdeConfig.FIND_LATEST_ARTIFACT, "true") @@ -369,7 +370,7 @@ void testTopicIdStrategyAutoRegister() throws Exception { new SimpleSerdesTesterBuilder().withTopic(topicName) .withSerializer(serializer).withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withDataGenerator(schema::generateMessage).withDataValidator(schema::validateMessage) - .withCommonProperty(SerdeConfig.ENABLE_HEADERS, "true") + .withCommonProperty(KafkaSerdeConfig.ENABLE_HEADERS, "true") .withProducerProperty(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true") .withAfterProduceValidator(() -> { return TestUtils.retry(() -> { @@ -436,7 +437,7 @@ public void testAutoRegisterDeriveClassProtobufType() throws Exception { .withSerializer(serializer).withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withDataGenerator(schema::generateMessage).withDataValidator(schema::validateMessage) .withProducerProperty(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true") - .withConsumerProperty(ProtobufKafkaDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") + .withConsumerProperty(ProtobufDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") .withAfterProduceValidator(() -> { return TestUtils.retry(() -> { VersionMetaData meta = registryClient.groups().byGroupId("default").artifacts() @@ -468,7 +469,7 @@ public void testAutoRegisterAndUseBody() throws Exception { .withSerializer(serializer).withDeserializer(deserializer).withStrategy(TopicIdStrategy.class) .withDataGenerator(schema::generateMessage).withDataValidator(schema::validateMessage) .withProducerProperty(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true") - .withConsumerProperty(ProtobufKafkaDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") + .withConsumerProperty(ProtobufDeserializerConfig.DERIVE_CLASS_FROM_SCHEMA, "true") .withAfterProduceValidator(() -> { return TestUtils.retry(() -> { VersionMetaData meta = registryClient.groups().byGroupId("default").artifacts() diff --git a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/SerdesTester.java b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/SerdesTester.java index 3daa0a4607..392011f26f 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/SerdesTester.java +++ b/integration-tests/src/test/java/io/apicurio/tests/serdes/apicurio/SerdesTester.java @@ -1,6 +1,6 @@ package io.apicurio.tests.serdes.apicurio; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.utils.tests.TestUtils; import io.apicurio.tests.ApicurioRegistryBaseIT; import io.apicurio.tests.utils.KafkaFacade; diff --git a/integration-tests/src/test/java/io/apicurio/tests/serdes/confluent/BasicConfluentSerDesIT.java b/integration-tests/src/test/java/io/apicurio/tests/serdes/confluent/BasicConfluentSerDesIT.java index 8b5f484989..a2c5f82f3b 100644 --- a/integration-tests/src/test/java/io/apicurio/tests/serdes/confluent/BasicConfluentSerDesIT.java +++ b/integration-tests/src/test/java/io/apicurio/tests/serdes/confluent/BasicConfluentSerDesIT.java @@ -105,7 +105,6 @@ void testAvroApicurioConfluent() throws Exception { new SimpleSerdesTesterBuilder().withTopic(topicName) .withSerializer(AvroKafkaSerializer.class) - // very important .withDeserializer(KafkaAvroDeserializer.class) .withStrategy(io.apicurio.registry.serde.strategy.TopicIdStrategy.class) diff --git a/pom.xml b/pom.xml index 2f4223cb04..132f3d36b4 100644 --- a/pom.xml +++ b/pom.xml @@ -80,10 +80,14 @@ utils/protobuf-schema-utilities utils/tests schema-resolver - serdes/serde-common - serdes/avro-serde - serdes/protobuf-serde - serdes/jsonschema-serde + serdes/generic/serde-common + serdes/generic/serde-common-avro + serdes/generic/serde-common-jsonschema + serdes/generic/serde-common-protobuf + serdes/kafka/serde-kafka-common + serdes/kafka/avro-serde + serdes/kafka/protobuf-serde + serdes/kafka/jsonschema-serde distro docs java-sdk @@ -333,17 +337,37 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-serde-common-avro ${project.version} io.apicurio - apicurio-registry-serdes-protobuf-serde + apicurio-registry-serde-common-jsonschema ${project.version} io.apicurio - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-serde-common-protobuf + ${project.version} + + + io.apicurio + apicurio-registry-serde-kafka-common + ${project.version} + + + io.apicurio + apicurio-registry-avro-serde-kafka + ${project.version} + + + io.apicurio + apicurio-registry-jsonschema-serde-kafka + ${project.version} + + + io.apicurio + apicurio-registry-protobuf-serde-kafka ${project.version} diff --git a/schema-resolver/src/main/java/io/apicurio/registry/resolver/config/AbstractConfig.java b/schema-resolver/src/main/java/io/apicurio/registry/resolver/config/AbstractConfig.java new file mode 100644 index 0000000000..1475bef7a4 --- /dev/null +++ b/schema-resolver/src/main/java/io/apicurio/registry/resolver/config/AbstractConfig.java @@ -0,0 +1,138 @@ +package io.apicurio.registry.resolver.config; + +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public abstract class AbstractConfig { + + protected Map originals; + + protected abstract Map getDefaults(); + + protected Duration getDurationNonNegativeMillis(String key) { + Object value = getObject(key); + if (value == null) { + reportError(key, "a non-null value", value); + } + long millis; + + if (value instanceof Number) { + millis = ((Number) value).longValue(); + } else if (value instanceof String) { + millis = Long.parseLong((String) value); + } else if (value instanceof Duration) { + millis = ((Duration) value).toMillis(); + } else { + reportError(key, "a duration-like value", value); + throw new IllegalStateException("Unreachable"); + } + if (millis < 0) { + reportError(key, "a non-negative duration-like value", value); + } + return Duration.ofMillis(millis); + } + + protected long getLongNonNegative(String key) { + Object value = getObject(key); + if (value == null) { + reportError(key, "a non-null value", value); + } + long result; + if (value instanceof Number) { + result = ((Number) value).longValue(); + } else if (value instanceof String) { + result = Long.parseLong((String) value); + } else { + reportError(key, "a number-like value", value); + throw new IllegalStateException("Unreachable"); + } + if (result < 0) { + reportError(key, "a non-negative number-like value", value); + } + return result; + } + + protected String getString(String key) { + Object value = getObject(key); + if (value == null) { + return null; + } + if (value instanceof String) { + return ((String) value).trim(); + } else { + reportError(key, "a String", value.getClass().getName()); + throw new IllegalStateException("Unreachable"); + } + } + + protected String getStringOneOf(String key, String... possibilities) { + String result = getString(key); + if (!Arrays.asList(possibilities).contains(result)) { + reportError(key, "one of " + Arrays.toString(possibilities), result); + } + return result; + } + + protected Boolean getBooleanOrFalse(String key) { + var val = getBoolean(key); + return val != null && val; + } + + protected Boolean getBoolean(String key) { + Object value = getObject(key); + if (value == null) { + return false; + } + if (value instanceof Boolean) { + return (Boolean) value; + } else if (value instanceof String) { + String trimmed = ((String) value).trim(); + if (trimmed.equalsIgnoreCase("true")) + return true; + else if (trimmed.equalsIgnoreCase("false")) + return false; + else { + reportError(key, "a boolean-like value", value); + throw new IllegalStateException("Unreachable"); + } + } else { + reportError(key, "a boolean-like value", value); + throw new IllegalStateException("Unreachable"); + } + } + + protected Object getObject(String key) { + if (key == null) { + throw new NullPointerException("Configuration property key is null."); + } + if (!originals.containsKey(key) && getDefaults().containsKey(key)) { + return getDefaults().get(key); + } + return originals.get(key); + } + + protected Class getClass(String className) { + try { + String originalsClassName = (String) this.originals.get(className); + if (originalsClassName != null) { + return this.getClass().getClassLoader().loadClass(originalsClassName); + } else { + return null; + } + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + public Map originals() { + return new HashMap<>(originals); + } + + private void reportError(String key, String expectedText, Object value) { + throw new IllegalArgumentException("Invalid configuration property value for '" + key + "'. " + + "Expected " + expectedText + ", but got a '" + value + "'."); + } + +} diff --git a/schema-resolver/src/main/java/io/apicurio/registry/resolver/config/DefaultSchemaResolverConfig.java b/schema-resolver/src/main/java/io/apicurio/registry/resolver/config/DefaultSchemaResolverConfig.java index 19f8d1e2ec..1bd1149e3c 100644 --- a/schema-resolver/src/main/java/io/apicurio/registry/resolver/config/DefaultSchemaResolverConfig.java +++ b/schema-resolver/src/main/java/io/apicurio/registry/resolver/config/DefaultSchemaResolverConfig.java @@ -1,14 +1,12 @@ package io.apicurio.registry.resolver.config; import java.time.Duration; -import java.util.Arrays; -import java.util.HashMap; import java.util.Map; import static io.apicurio.registry.resolver.SchemaResolverConfig.*; import static java.util.Map.entry; -public class DefaultSchemaResolverConfig { +public class DefaultSchemaResolverConfig extends AbstractConfig { private static final Map DEFAULTS = Map.ofEntries( entry(ARTIFACT_RESOLVER_STRATEGY, ARTIFACT_RESOLVER_STRATEGY_DEFAULT), @@ -22,8 +20,6 @@ public class DefaultSchemaResolverConfig { entry(DEREFERENCE_SCHEMA, DEREFERENCE_SCHEMA_DEFAULT), entry(DESERIALIZER_DEREFERENCE_SCHEMA, DESERIALIZER_DEREFERENCE_SCHEMA_DEFAULT)); - private Map originals; - public DefaultSchemaResolverConfig(Map originals) { this.originals = originals; } @@ -119,20 +115,6 @@ public String getExplicitArtifactVersion() { return getString(EXPLICIT_ARTIFACT_VERSION); } - public Map originals() { - return new HashMap<>(originals); - } - - Object getObject(String key) { - if (key == null) { - throw new NullPointerException("Configuration property key is null."); - } - if (!originals.containsKey(key) && DEFAULTS.containsKey(key)) { - return DEFAULTS.get(key); - } - return originals.get(key); - } - public boolean serializerDereference() { return getBooleanOrFalse(DEREFERENCE_SCHEMA); } @@ -141,100 +123,8 @@ public boolean deserializerDereference() { return getBooleanOrFalse(DESERIALIZER_DEREFERENCE_SCHEMA); } - private Duration getDurationNonNegativeMillis(String key) { - Object value = getObject(key); - if (value == null) { - reportError(key, "a non-null value", value); - } - long millis; - - if (value instanceof Number) { - millis = ((Number) value).longValue(); - } else if (value instanceof String) { - millis = Long.parseLong((String) value); - } else if (value instanceof Duration) { - millis = ((Duration) value).toMillis(); - } else { - reportError(key, "a duration-like value", value); - throw new IllegalStateException("Unreachable"); - } - if (millis < 0) { - reportError(key, "a non-negative duration-like value", value); - } - return Duration.ofMillis(millis); - } - - private long getLongNonNegative(String key) { - Object value = getObject(key); - if (value == null) { - reportError(key, "a non-null value", value); - } - long result; - if (value instanceof Number) { - result = ((Number) value).longValue(); - } else if (value instanceof String) { - result = Long.parseLong((String) value); - } else { - reportError(key, "a number-like value", value); - throw new IllegalStateException("Unreachable"); - } - if (result < 0) { - reportError(key, "a non-negative number-like value", value); - } - return result; - } - - private String getString(String key) { - Object value = getObject(key); - if (value == null) { - return null; - } - if (value instanceof String) { - return ((String) value).trim(); - } else { - reportError(key, "a String", value.getClass().getName()); - throw new IllegalStateException("Unreachable"); - } - } - - private String getStringOneOf(String key, String... possibilities) { - String result = getString(key); - if (!Arrays.asList(possibilities).contains(result)) { - reportError(key, "one of " + Arrays.toString(possibilities), result); - } - return result; - } - - private Boolean getBooleanOrFalse(String key) { - var val = getBoolean(key); - return val != null && val; - } - - private Boolean getBoolean(String key) { - Object value = getObject(key); - if (value == null) { - return null; - } - if (value instanceof Boolean) { - return (Boolean) value; - } else if (value instanceof String) { - String trimmed = ((String) value).trim(); - if (trimmed.equalsIgnoreCase("true")) - return true; - else if (trimmed.equalsIgnoreCase("false")) - return false; - else { - reportError(key, "a boolean-like value", value); - throw new IllegalStateException("Unreachable"); - } - } else { - reportError(key, "a boolean-like value", value); - throw new IllegalStateException("Unreachable"); - } - } - - private void reportError(String key, String expectedText, Object value) { - throw new IllegalArgumentException("Invalid configuration property value for '" + key + "'. " - + "Expected " + expectedText + ", but got a '" + value + "'."); + @Override + protected Map getDefaults() { + return DEFAULTS; } } diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerde.java b/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerde.java deleted file mode 100644 index 5f37a096d5..0000000000 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerde.java +++ /dev/null @@ -1,12 +0,0 @@ -package io.apicurio.registry.serde.avro; - -import io.apicurio.registry.serde.AbstractSerde; - -/** - * Wraps the AvroKafkaSerializer and AvroKafkaDeserializer. - */ -public class AvroSerde extends AbstractSerde { - public AvroSerde() { - super(new AvroKafkaSerializer<>(), new AvroKafkaDeserializer<>()); - } -} diff --git a/serdes/avro-serde/pom.xml b/serdes/generic/serde-common-avro/pom.xml similarity index 77% rename from serdes/avro-serde/pom.xml rename to serdes/generic/serde-common-avro/pom.xml index 35b7a87462..f8f0540f99 100644 --- a/serdes/avro-serde/pom.xml +++ b/serdes/generic/serde-common-avro/pom.xml @@ -5,15 +5,15 @@ io.apicurio apicurio-registry 3.0.0-SNAPSHOT - ../../pom.xml + ../../../pom.xml - apicurio-registry-serdes-avro-serde + apicurio-registry-serde-common-avro jar - apicurio-registry-serdes-avro-serde + apicurio-registry-serde-avro-common - ${project.basedir}/../.. + ${project.basedir}/../../.. diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroDatumProvider.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroDatumProvider.java similarity index 85% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroDatumProvider.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroDatumProvider.java index aae8aa380d..fe9e294391 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroDatumProvider.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroDatumProvider.java @@ -6,7 +6,7 @@ public interface AvroDatumProvider { - default void configure(AvroKafkaSerdeConfig config) { + default void configure(AvroSerdeConfig config) { } DatumWriter createDatumWriter(T data, Schema schema); diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaDeserializer.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroDeserializer.java similarity index 63% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaDeserializer.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroDeserializer.java index 2eaee94041..dc39c1fda4 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaDeserializer.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroDeserializer.java @@ -2,56 +2,62 @@ import io.apicurio.registry.resolver.ParsedSchema; import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import io.apicurio.registry.resolver.utils.Utils; import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.AbstractKafkaDeserializer; +import io.apicurio.registry.serde.AbstractDeserializer; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.avro.Schema; import org.apache.avro.io.DatumReader; import org.apache.avro.io.DecoderFactory; -import org.apache.kafka.common.header.Headers; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; -import java.util.Map; import java.util.Objects; import java.util.function.Consumer; -public class AvroKafkaDeserializer extends AbstractKafkaDeserializer { +public class AvroDeserializer extends AbstractDeserializer { private final DecoderFactory decoderFactory = DecoderFactory.get(); private AvroSchemaParser parser; private AvroDatumProvider avroDatumProvider; - private AvroEncoding configEncoding; - private AvroSerdeHeaders avroHeaders; + private AvroEncoding encoding; - public AvroKafkaDeserializer() { + public AvroDeserializer() { super(); } - public AvroKafkaDeserializer(RegistryClient client) { + public AvroDeserializer(RegistryClient client) { super(client); } - private AvroKafkaDeserializer setAvroDatumProvider(AvroDatumProvider avroDatumProvider) { - this.avroDatumProvider = Objects.requireNonNull(avroDatumProvider); - return this; + public AvroDeserializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public AvroDeserializer(RegistryClient client, SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public AvroDeserializer(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + super(client, strategy, schemaResolver); } @SuppressWarnings("rawtypes") @Override - public void configure(Map configs, boolean isKey) { - AvroKafkaSerdeConfig config = new AvroKafkaSerdeConfig(configs); - configEncoding = config.getAvroEncoding(); + public void configure(SerdeConfig configs, boolean isKey) { + AvroSerdeConfig config = new AvroSerdeConfig(configs.originals()); + encoding = config.getAvroEncoding(); Class adp = config.getAvroDatumProvider(); Consumer consumer = this::setAvroDatumProvider; Utils.instantiate(AvroDatumProvider.class, adp, consumer); avroDatumProvider.configure(config); - avroHeaders = new AvroSerdeHeaders(isKey); - // important to instantiate the SchemaParser before calling super.configure parser = new AvroSchemaParser<>(avroDatumProvider); @@ -59,7 +65,7 @@ public void configure(Map configs, boolean isKey) { } /** - * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + * @see io.apicurio.registry.serde.AbstractSerDe#schemaParser() */ @Override public SchemaParser schemaParser() { @@ -68,22 +74,10 @@ public SchemaParser schemaParser() { @Override protected U readData(ParsedSchema schema, ByteBuffer buffer, int start, int length) { - return readData(null, schema, buffer, start, length); - } - - @Override - protected U readData(Headers headers, ParsedSchema schema, ByteBuffer buffer, int start, - int length) { AvroEncoding encoding = null; - if (headers != null) { - String encodingHeader = avroHeaders.getEncoding(headers); - if (encodingHeader != null) { - encoding = AvroEncoding.valueOf(encodingHeader); - } - } - if (encoding == null) { + if (this.encoding != null) { // no encoding in header or no headers so use config - encoding = configEncoding; + encoding = this.encoding; } try { DatumReader reader = avroDatumProvider.createDatumReader(schema.getParsedSchema()); @@ -101,4 +95,13 @@ protected U readData(Headers headers, ParsedSchema schema, ByteBuffer bu throw new UncheckedIOException(e); } } + + public AvroDeserializer setAvroDatumProvider(AvroDatumProvider avroDatumProvider) { + this.avroDatumProvider = Objects.requireNonNull(avroDatumProvider); + return this; + } + + public void setEncoding(AvroEncoding encoding) { + this.encoding = encoding; + } } diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroEncoding.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroEncoding.java similarity index 100% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroEncoding.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroEncoding.java diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaParser.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaParser.java similarity index 97% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaParser.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaParser.java index 0059d2ce53..ea0cf095e3 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaParser.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaParser.java @@ -23,7 +23,7 @@ public AvroSchemaParser(AvroDatumProvider avroDatumProvider) { } /** - * @see io.apicurio.registry.serde.SchemaParser#artifactType() + * @see io.apicurio.registry.resolver.SchemaParser#artifactType() */ @Override public String artifactType() { @@ -31,7 +31,7 @@ public String artifactType() { } /** - * @see io.apicurio.registry.serde.SchemaParser#parseSchema(byte[]) + * @see io.apicurio.registry.resolver.SchemaParser#parseSchema(byte[], Map>) */ @Override public Schema parseSchema(byte[] rawSchema, Map> resolvedReferences) { diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaUtils.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaUtils.java similarity index 96% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaUtils.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaUtils.java index eef5d172c1..9075526dd3 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaUtils.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSchemaUtils.java @@ -6,7 +6,6 @@ import org.apache.avro.SchemaParseException; import org.apache.avro.generic.GenericContainer; import org.apache.avro.reflect.ReflectData; -import org.apache.kafka.common.errors.SerializationException; import java.nio.ByteBuffer; import java.util.Collections; @@ -73,7 +72,7 @@ static Schema getReflectSchema(ReflectData reflectData, Object object) { Class clazz = (object instanceof Class) ? (Class) object : object.getClass(); Schema schema = reflectData.getSchema(clazz); if (schema == null) { - throw new SerializationException("No schema for class: " + clazz.getName()); + throw new IllegalStateException("No schema for class: " + clazz.getName()); } return schema; } @@ -102,7 +101,7 @@ static Schema getSchema(Object object, boolean useReflection) { } else if (useReflection) { Schema schema = ReflectData.get().getSchema(object.getClass()); if (schema == null) { - throw new SerializationException( + throw new IllegalStateException( "Schema is null for object of class " + object.getClass().getCanonicalName()); } else { return schema; diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerdeConfig.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeConfig.java similarity index 54% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerdeConfig.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeConfig.java index 05e78b398d..91f60278b6 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerdeConfig.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeConfig.java @@ -1,13 +1,11 @@ package io.apicurio.registry.serde.avro; -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; +import io.apicurio.registry.serde.config.SerdeConfig; +import java.util.HashMap; import java.util.Map; -public class AvroKafkaSerdeConfig extends BaseKafkaSerDeConfig { +public class AvroSerdeConfig extends SerdeConfig { /** * Used by the Avro serde classes to choose an io.apicurio.registry.serde.avro.AvroEncoding, @@ -24,25 +22,10 @@ public class AvroKafkaSerdeConfig extends BaseKafkaSerDeConfig { public static final String USE_SPECIFIC_AVRO_READER = "apicurio.registry.use-specific-avro-reader"; public static final boolean USE_SPECIFIC_AVRO_READER_DEFAULT = false; - private static ConfigDef configDef() { - ConfigDef configDef = new ConfigDef() - .define(AVRO_ENCODING, Type.STRING, AvroEncoding.BINARY.name(), Importance.MEDIUM, - "TODO docs") - .define(AVRO_DATUM_PROVIDER, Type.CLASS, AVRO_DATUM_PROVIDER_DEFAULT, Importance.MEDIUM, - "TODO docs") - .define(USE_SPECIFIC_AVRO_READER, Type.BOOLEAN, USE_SPECIFIC_AVRO_READER_DEFAULT, - Importance.MEDIUM, "TODO docs"); - return configDef; - } - - /** - * Constructor. - * - * @param configDef - * @param originals - */ - public AvroKafkaSerdeConfig(Map originals) { - super(configDef(), originals); + public AvroSerdeConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; } public AvroEncoding getAvroEncoding() { @@ -56,4 +39,15 @@ public Class getAvroDatumProvider() { public boolean useSpecificAvroReader() { return this.getBoolean(USE_SPECIFIC_AVRO_READER); } + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } + + private static final Map DEFAULTS = Map.of(AVRO_ENCODING, AvroEncoding.BINARY.name(), + AVRO_DATUM_PROVIDER, AVRO_DATUM_PROVIDER_DEFAULT, USE_SPECIFIC_AVRO_READER, + USE_SPECIFIC_AVRO_READER_DEFAULT); } diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerializer.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSerializer.java similarity index 64% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerializer.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSerializer.java index ab6bbc2040..688670f67c 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerializer.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/AvroSerializer.java @@ -6,54 +6,60 @@ import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import io.apicurio.registry.resolver.utils.Utils; import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.AbstractKafkaSerializer; +import io.apicurio.registry.serde.AbstractSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; import org.apache.avro.Schema; import org.apache.avro.io.DatumWriter; import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; -import org.apache.kafka.common.header.Headers; import java.io.IOException; import java.io.OutputStream; -import java.util.Map; import java.util.Objects; import java.util.function.Consumer; -public class AvroKafkaSerializer extends AbstractKafkaSerializer { +public class AvroSerializer extends AbstractSerializer { private final EncoderFactory encoderFactory = EncoderFactory.get(); private AvroSchemaParser parser; private AvroDatumProvider avroDatumProvider; private AvroEncoding encoding; - private AvroSerdeHeaders avroHeaders; - public AvroKafkaSerializer() { + public AvroSerializer() { super(); } - public AvroKafkaSerializer(RegistryClient client) { + public AvroSerializer(RegistryClient client) { super(client); } - public AvroKafkaSerializer(SchemaResolver schemaResolver) { + public AvroSerializer(SchemaResolver schemaResolver) { super(schemaResolver); } - public AvroKafkaSerializer(RegistryClient client, + public AvroSerializer(RegistryClient client, ArtifactReferenceResolverStrategy artifactResolverStrategy, SchemaResolver schemaResolver) { super(client, artifactResolverStrategy, schemaResolver); } - private AvroKafkaSerializer setAvroDatumProvider(AvroDatumProvider avroDatumProvider) { + private AvroSerializer setAvroDatumProvider(AvroDatumProvider avroDatumProvider) { this.avroDatumProvider = Objects.requireNonNull(avroDatumProvider); return this; } + public void setEncoding(AvroEncoding encoding) { + this.encoding = encoding; + } + + public AvroEncoding getEncoding() { + return this.encoding; + } + @SuppressWarnings("rawtypes") @Override - public void configure(Map configs, boolean isKey) { - AvroKafkaSerdeConfig config = new AvroKafkaSerdeConfig(configs); + public void configure(SerdeConfig configs, boolean isKey) { + AvroSerdeConfig config = new AvroSerdeConfig(configs.originals()); encoding = config.getAvroEncoding(); Class adp = config.getAvroDatumProvider(); @@ -61,8 +67,6 @@ public void configure(Map configs, boolean isKey) { Utils.instantiate(AvroDatumProvider.class, adp, consumer); avroDatumProvider.configure(config); - avroHeaders = new AvroSerdeHeaders(isKey); - // important to instantiate the SchemaParser before calling super.configure parser = new AvroSchemaParser<>(avroDatumProvider); @@ -70,7 +74,7 @@ public void configure(Map configs, boolean isKey) { } /** - * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + * @see io.apicurio.registry.serde.AbstractSerDe#schemaParser() */ @Override public SchemaParser schemaParser() { @@ -78,7 +82,7 @@ public SchemaParser schemaParser() { } /** - * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(io.apicurio.registry.serde.ParsedSchema, + * @see io.apicurio.registry.serde.AbstractSerializer#serializeData(io.apicurio.registry.resolver.ParsedSchema, * java.lang.Object, java.io.OutputStream) */ @SuppressWarnings("unchecked") @@ -86,7 +90,6 @@ public SchemaParser schemaParser() { protected void serializeData(ParsedSchema schema, U data, OutputStream out) throws IOException { Encoder encoder = createEncoder(schema.getParsedSchema(), out); - // I guess this can happen if generics are lost with reflection ... if (data instanceof NonRecordContainer) { // noinspection unchecked data = (U) NonRecordContainer.class.cast(data).getValue(); @@ -97,19 +100,6 @@ protected void serializeData(ParsedSchema schema, U data, OutputStream o encoder.flush(); } - /** - * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(org.apache.kafka.common.header.Headers, - * io.apicurio.registry.serde.ParsedSchema, java.lang.Object, java.io.OutputStream) - */ - @Override - protected void serializeData(Headers headers, ParsedSchema schema, U data, OutputStream out) - throws IOException { - if (headers != null) { - avroHeaders.addEncodingHeader(headers, encoding.name()); - } - serializeData(schema, data, out); - } - private Encoder createEncoder(Schema schema, OutputStream os) throws IOException { if (encoding == AvroEncoding.JSON) { return encoderFactory.jsonEncoder(schema, os); diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/DefaultAvroDatumProvider.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/DefaultAvroDatumProvider.java similarity index 98% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/DefaultAvroDatumProvider.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/DefaultAvroDatumProvider.java index 858e2313b0..57583a3841 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/DefaultAvroDatumProvider.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/DefaultAvroDatumProvider.java @@ -30,7 +30,7 @@ public DefaultAvroDatumProvider setUseSpecificAvroReader(boolean useSpecificA } @Override - public void configure(AvroKafkaSerdeConfig config) { + public void configure(AvroSerdeConfig config) { if (useSpecificAvroReader == null) { useSpecificAvroReader = config.useSpecificAvroReader(); } diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/NonRecordContainer.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/NonRecordContainer.java similarity index 89% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/NonRecordContainer.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/NonRecordContainer.java index d17d50a8b3..041ef52a1f 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/NonRecordContainer.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/NonRecordContainer.java @@ -2,7 +2,6 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericContainer; -import org.apache.kafka.common.errors.SerializationException; import java.util.Objects; @@ -16,7 +15,7 @@ public class NonRecordContainer implements GenericContainer { public NonRecordContainer(Schema schema, T value) { if (schema == null) { - throw new SerializationException("Schema may not be null."); + throw new IllegalStateException("Schema may not be null."); } this.schema = schema; this.value = value; diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/ReflectAllowNullAvroDatumProvider.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/ReflectAllowNullAvroDatumProvider.java similarity index 100% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/ReflectAllowNullAvroDatumProvider.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/ReflectAllowNullAvroDatumProvider.java diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/ReflectAvroDatumProvider.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/ReflectAvroDatumProvider.java similarity index 100% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/ReflectAvroDatumProvider.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/ReflectAvroDatumProvider.java diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/strategy/QualifiedRecordIdStrategy.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/strategy/QualifiedRecordIdStrategy.java similarity index 88% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/strategy/QualifiedRecordIdStrategy.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/strategy/QualifiedRecordIdStrategy.java index 8909f55ba0..f1ba149f0a 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/strategy/QualifiedRecordIdStrategy.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/strategy/QualifiedRecordIdStrategy.java @@ -5,7 +5,6 @@ import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import io.apicurio.registry.serde.strategy.ArtifactReference; import org.apache.avro.Schema; -import org.apache.kafka.common.errors.SerializationException; public class QualifiedRecordIdStrategy implements ArtifactReferenceResolverStrategy { @@ -22,7 +21,7 @@ public io.apicurio.registry.resolver.strategy.ArtifactReference artifactReferenc return ArtifactReference.builder().groupId(null) .artifactId(parsedSchema.getParsedSchema().getFullName()).build(); } - throw new SerializationException("The message must only be an Avro record schema!"); + throw new IllegalStateException("The message must only be an Avro record schema!"); } } diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/strategy/RecordIdStrategy.java b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/strategy/RecordIdStrategy.java similarity index 87% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/strategy/RecordIdStrategy.java rename to serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/strategy/RecordIdStrategy.java index 0c2d3f9a96..50de9cf77f 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/strategy/RecordIdStrategy.java +++ b/serdes/generic/serde-common-avro/src/main/java/io/apicurio/registry/serde/avro/strategy/RecordIdStrategy.java @@ -5,7 +5,6 @@ import io.apicurio.registry.resolver.strategy.ArtifactReference; import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import org.apache.avro.Schema; -import org.apache.kafka.common.errors.SerializationException; public class RecordIdStrategy implements ArtifactReferenceResolverStrategy { @@ -21,7 +20,7 @@ public ArtifactReference artifactReference(Record data, ParsedSchema data, ParsedSchema parsedSchema) { ArtifactReference reference = super.artifactReference(data, parsedSchema); - KafkaSerdeRecord kdata = (KafkaSerdeRecord) data; + SerdeRecord kdata = (SerdeRecord) data; return ArtifactReference.builder().groupId(reference.getGroupId()) .artifactId(kdata.metadata().getTopic() + "-" + reference.getArtifactId()) .version(reference.getVersion()).build(); diff --git a/serdes/jsonschema-serde/pom.xml b/serdes/generic/serde-common-jsonschema/pom.xml similarity index 81% rename from serdes/jsonschema-serde/pom.xml rename to serdes/generic/serde-common-jsonschema/pom.xml index 97ab490b3d..b8e236569d 100644 --- a/serdes/jsonschema-serde/pom.xml +++ b/serdes/generic/serde-common-jsonschema/pom.xml @@ -5,15 +5,15 @@ io.apicurio apicurio-registry 3.0.0-SNAPSHOT - ../../pom.xml + ../../../pom.xml - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-serde-common-jsonschema jar - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-serde-common-jsonschema - ${project.basedir}/../.. + ${project.basedir}/../../.. diff --git a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaDeserializer.java similarity index 61% rename from serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java rename to serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaDeserializer.java index b14aab1fb5..b28b191d2d 100644 --- a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java +++ b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaDeserializer.java @@ -11,58 +11,52 @@ import io.apicurio.registry.resolver.SchemaResolver; import io.apicurio.registry.resolver.utils.Utils; import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.AbstractKafkaDeserializer; -import io.apicurio.registry.serde.headers.MessageTypeSerdeHeaders; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.common.serialization.Deserializer; +import io.apicurio.registry.serde.AbstractDeserializer; +import io.apicurio.registry.serde.config.SerdeConfig; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -public class JsonSchemaKafkaDeserializer extends AbstractKafkaDeserializer - implements Deserializer { +public class JsonSchemaDeserializer extends AbstractDeserializer { private ObjectMapper mapper; private Boolean validationEnabled; - private JsonSchemaParser parser = new JsonSchemaParser<>(); + private final JsonSchemaParser parser = new JsonSchemaParser<>(); /** * Optional, the full class name of the java class to deserialize */ private Class specificReturnClass; - private MessageTypeSerdeHeaders serdeHeaders; - public JsonSchemaKafkaDeserializer() { + public JsonSchemaDeserializer() { super(); } - public JsonSchemaKafkaDeserializer(RegistryClient client, SchemaResolver schemaResolver) { + public JsonSchemaDeserializer(RegistryClient client, SchemaResolver schemaResolver) { super(client, schemaResolver); } - public JsonSchemaKafkaDeserializer(RegistryClient client) { + public JsonSchemaDeserializer(RegistryClient client) { super(client); } - public JsonSchemaKafkaDeserializer(SchemaResolver schemaResolver) { + public JsonSchemaDeserializer(SchemaResolver schemaResolver) { super(schemaResolver); } - public JsonSchemaKafkaDeserializer(RegistryClient client, Boolean validationEnabled) { + public JsonSchemaDeserializer(RegistryClient client, Boolean validationEnabled) { this(client); this.validationEnabled = validationEnabled; } /** - * @see io.apicurio.registry.serde.AbstractKafkaDeserializer#configure(java.util.Map, boolean) + * @see AbstractDeserializer#configure(SerdeConfig, boolean) */ @SuppressWarnings("unchecked") @Override - public void configure(Map configs, boolean isKey) { - JsonSchemaKafkaDeserializerConfig config = new JsonSchemaKafkaDeserializerConfig(configs, isKey); + public void configure(SerdeConfig configs, boolean isKey) { + JsonSchemaDeserializerConfig config = new JsonSchemaDeserializerConfig(configs.originals(), isKey); super.configure(config, isKey); if (validationEnabled == null) { @@ -71,8 +65,6 @@ public void configure(Map configs, boolean isKey) { this.specificReturnClass = (Class) config.getSpecificReturnClass(); - this.serdeHeaders = new MessageTypeSerdeHeaders(new HashMap<>(configs), isKey); - if (null == mapper) { mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .setSerializationInclusion(JsonInclude.Include.NON_NULL); @@ -88,8 +80,16 @@ public void setObjectMapper(ObjectMapper objectMapper) { this.mapper = objectMapper; } + public void setSpecificReturnClass(Class specificReturnClass) { + this.specificReturnClass = specificReturnClass; + } + + public Class getSpecificReturnClass() { + return specificReturnClass; + } + /** - * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + * @see io.apicurio.registry.serde.AbstractSerDe#schemaParser() */ @Override public SchemaParser schemaParser() { @@ -97,26 +97,15 @@ public SchemaParser schemaParser() { } /** - * @see io.apicurio.registry.serde.AbstractKafkaDeserializer#readData(io.apicurio.registry.serde.ParsedSchema, - * java.nio.ByteBuffer, int, int) + * @see AbstractDeserializer#readData(io.apicurio.registry.resolver.ParsedSchema, java.nio.ByteBuffer, + * int, int) */ @Override protected T readData(ParsedSchema schema, ByteBuffer buffer, int start, int length) { - return internalReadData(null, schema, buffer, start, length); + return internalReadData(schema, buffer, start, length); } - /** - * @see io.apicurio.registry.serde.AbstractKafkaDeserializer#readData(org.apache.kafka.common.header.Headers, - * io.apicurio.registry.serde.ParsedSchema, java.nio.ByteBuffer, int, int) - */ - @Override - protected T readData(Headers headers, ParsedSchema schema, ByteBuffer buffer, int start, - int length) { - return internalReadData(headers, schema, buffer, start, length); - } - - private T internalReadData(Headers headers, ParsedSchema schema, ByteBuffer buffer, int start, - int length) { + private T internalReadData(ParsedSchema schema, ByteBuffer buffer, int start, int length) { byte[] data = new byte[length]; System.arraycopy(buffer.array(), start, data, 0, length); @@ -131,7 +120,7 @@ private T internalReadData(Headers headers, ParsedSchema schema, Byt if (this.specificReturnClass != null) { messageType = this.specificReturnClass; - } else if (headers == null) { + } else { JsonNode jsonSchema = mapper.readTree(schema.getRawSchema()); String javaType = null; @@ -143,9 +132,6 @@ private T internalReadData(Headers headers, ParsedSchema schema, Byt // You can try configure the property \"apicurio.registry.serde.json-schema.java-type\" with // the full class name to use for deserialization messageType = javaType == null ? null : Utils.loadClass(javaType); - } else { - String javaType = serdeHeaders.getMessageType(headers); - messageType = javaType == null ? null : Utils.loadClass(javaType); } if (messageType == null) { diff --git a/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaDeserializerConfig.java b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaDeserializerConfig.java new file mode 100644 index 0000000000..fc22d9d597 --- /dev/null +++ b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaDeserializerConfig.java @@ -0,0 +1,50 @@ +package io.apicurio.registry.serde.jsonschema; + +import io.apicurio.registry.serde.config.SerdeConfig; + +import java.util.HashMap; +import java.util.Map; + +public class JsonSchemaDeserializerConfig extends SerdeConfig { + + private boolean isKey; + + /** + * Constructor. + * + * @param originals + */ + public JsonSchemaDeserializerConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + } + + public JsonSchemaDeserializerConfig(Map originals, boolean isKey) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + this.isKey = isKey; + } + + public Class getSpecificReturnClass() { + if (isKey) { + return this.getClass(DESERIALIZER_SPECIFIC_KEY_RETURN_CLASS); + } else { + return this.getClass(DESERIALIZER_SPECIFIC_VALUE_RETURN_CLASS); + } + } + + public boolean validationEnabled() { + return this.getBoolean(VALIDATION_ENABLED); + } + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } + + private static final Map DEFAULTS = Map.of(VALIDATION_ENABLED, VALIDATION_ENABLED_DEFAULT); +} diff --git a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaParser.java b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaParser.java similarity index 93% rename from serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaParser.java rename to serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaParser.java index 0fdaea97a8..9f20ac0fa3 100644 --- a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaParser.java +++ b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaParser.java @@ -16,7 +16,7 @@ public class JsonSchemaParser implements SchemaParser { /** - * @see io.apicurio.registry.serde.SchemaParser#artifactType() + * @see io.apicurio.registry.resolver.SchemaParser#artifactType() */ @Override public String artifactType() { @@ -24,7 +24,8 @@ public String artifactType() { } /** - * @see io.apicurio.registry.serde.SchemaParser#parseSchema(byte[]) + * @see io.apicurio.registry.resolver.SchemaParser#parseSchema(byte[], Map>) */ @Override public JsonSchema parseSchema(byte[] rawSchema, @@ -54,7 +55,7 @@ private void resolveReferences(Map> resolvedRef } /** - * @see io.apicurio.registry.resolver.SchemaParser#getSchemaFromData(java.lang.Object) + * @see io.apicurio.registry.resolver.SchemaParser#getSchemaFromData(Record) */ @Override public ParsedSchema getSchemaFromData(Record data) { diff --git a/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerializer.java b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerializer.java new file mode 100644 index 0000000000..28561f534c --- /dev/null +++ b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerializer.java @@ -0,0 +1,111 @@ +package io.apicurio.registry.serde.jsonschema; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.networknt.schema.JsonSchema; +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.AbstractSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * An implementation of the Kafka Serializer for JSON Schema use-cases. This serializer assumes that the + * user's application needs to serialize a Java Bean to JSON data using Jackson. In addition to standard + * serialization of the bean, this implementation can also optionally validate it against a JSON schema. + */ +public class JsonSchemaSerializer extends AbstractSerializer { + + private ObjectMapper mapper; + private final JsonSchemaParser parser = new JsonSchemaParser<>(); + + private Boolean validationEnabled; + + public JsonSchemaSerializer() { + super(); + } + + public JsonSchemaSerializer(RegistryClient client, + ArtifactReferenceResolverStrategy artifactResolverStrategy, + SchemaResolver schemaResolver) { + super(client, artifactResolverStrategy, schemaResolver); + } + + public JsonSchemaSerializer(RegistryClient client) { + super(client); + } + + public JsonSchemaSerializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public JsonSchemaSerializer(RegistryClient client, Boolean validationEnabled) { + this(client); + this.validationEnabled = validationEnabled; + } + + /** + * @see io.apicurio.registry.serde.AbstractSerializer#configure(SerdeConfig, boolean) + */ + @Override + public void configure(SerdeConfig configs, boolean isKey) { + JsonSchemaSerializerConfig config = new JsonSchemaSerializerConfig(configs.originals()); + + if (validationEnabled == null) { + this.validationEnabled = config.validationEnabled(); + } + + if (null == mapper) { + this.mapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + super.configure(config, isKey); + } + + public boolean isValidationEnabled() { + return validationEnabled != null && validationEnabled; + } + + public void setObjectMapper(ObjectMapper objectMapper) { + this.mapper = objectMapper; + } + + /** + * @param validationEnabled the validationEnabled to set + */ + public void setValidationEnabled(Boolean validationEnabled) { + this.validationEnabled = validationEnabled; + } + + /** + * @see io.apicurio.registry.serde.AbstractSerDe#schemaParser() + */ + @Override + public SchemaParser schemaParser() { + return parser; + } + + /** + * @see io.apicurio.registry.serde.AbstractSerializer#serializeData(io.apicurio.registry.resolver.ParsedSchema, + * java.lang.Object, java.io.OutputStream) + */ + @Override + protected void serializeData(ParsedSchema schema, T data, OutputStream out) + throws IOException { + final byte[] dataBytes = mapper.writeValueAsBytes(data); + + if (isValidationEnabled()) { + JsonSchemaValidationUtil.validateDataWithSchema(schema, dataBytes, mapper); + } + + out.write(dataBytes); + } +} diff --git a/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerializerConfig.java b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerializerConfig.java new file mode 100644 index 0000000000..01f3b2f410 --- /dev/null +++ b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerializerConfig.java @@ -0,0 +1,33 @@ +package io.apicurio.registry.serde.jsonschema; + +import io.apicurio.registry.serde.config.SerdeConfig; + +import java.util.HashMap; +import java.util.Map; + +public class JsonSchemaSerializerConfig extends SerdeConfig { + + /** + * Constructor. + * + * @param originals + */ + public JsonSchemaSerializerConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + } + + public boolean validationEnabled() { + return this.getBoolean(VALIDATION_ENABLED); + } + + private static final Map DEFAULTS = Map.of(VALIDATION_ENABLED, VALIDATION_ENABLED_DEFAULT); + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } +} diff --git a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaValidationUtil.java b/serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaValidationUtil.java similarity index 100% rename from serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaValidationUtil.java rename to serdes/generic/serde-common-jsonschema/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaValidationUtil.java diff --git a/serdes/protobuf-serde/.gitignore b/serdes/generic/serde-common-protobuf/.gitignore similarity index 100% rename from serdes/protobuf-serde/.gitignore rename to serdes/generic/serde-common-protobuf/.gitignore diff --git a/serdes/protobuf-serde/pom.xml b/serdes/generic/serde-common-protobuf/pom.xml similarity index 90% rename from serdes/protobuf-serde/pom.xml rename to serdes/generic/serde-common-protobuf/pom.xml index 0f00d18dce..eb0d7a4630 100644 --- a/serdes/protobuf-serde/pom.xml +++ b/serdes/generic/serde-common-protobuf/pom.xml @@ -5,18 +5,19 @@ io.apicurio apicurio-registry 3.0.0-SNAPSHOT - ../../pom.xml + ../../../pom.xml - apicurio-registry-serdes-protobuf-serde + apicurio-registry-serde-common-protobuf jar - apicurio-registry-serdes-protobuf-serde + apicurio-registry-serde-common-protobuf - ${project.basedir}/../.. + ${project.basedir}/../../.. + io.apicurio apicurio-registry-serde-common diff --git a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializer.java b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufDeserializer.java similarity index 67% rename from serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializer.java rename to serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufDeserializer.java index e7bbd95b9d..2b05c07e67 100644 --- a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializer.java +++ b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufDeserializer.java @@ -10,12 +10,10 @@ import io.apicurio.registry.resolver.SchemaResolver; import io.apicurio.registry.resolver.utils.Utils; import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.AbstractKafkaDeserializer; +import io.apicurio.registry.serde.AbstractDeserializer; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.protobuf.ref.RefOuterClass.Ref; import io.apicurio.registry.utils.protobuf.schema.ProtobufSchema; -import org.apache.kafka.common.config.ConfigException; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.header.Headers; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -24,12 +22,10 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -public class ProtobufKafkaDeserializer - extends AbstractKafkaDeserializer { +public class ProtobufDeserializer extends AbstractDeserializer { private static final String PROTOBUF_PARSE_METHOD = "parseFrom"; @@ -38,31 +34,29 @@ public class ProtobufKafkaDeserializer private Class specificReturnClass; private Method specificReturnClassParseMethod; private boolean deriveClass; + private String messageTypeName; - private Map parseMethodsCache = new ConcurrentHashMap<>(); + private final Map parseMethodsCache = new ConcurrentHashMap<>(); - private ProtobufSerdeHeaders serdeHeaders; - - public ProtobufKafkaDeserializer() { + public ProtobufDeserializer() { super(); } - public ProtobufKafkaDeserializer(RegistryClient client, - SchemaResolver schemaResolver) { + public ProtobufDeserializer(RegistryClient client, SchemaResolver schemaResolver) { super(client, schemaResolver); } - public ProtobufKafkaDeserializer(RegistryClient client) { + public ProtobufDeserializer(RegistryClient client) { super(client); } - public ProtobufKafkaDeserializer(SchemaResolver schemaResolver) { + public ProtobufDeserializer(SchemaResolver schemaResolver) { super(schemaResolver); } @Override - public void configure(Map configs, boolean isKey) { - ProtobufKafkaDeserializerConfig config = new ProtobufKafkaDeserializerConfig(configs, isKey); + public void configure(SerdeConfig configs, boolean isKey) { + ProtobufDeserializerConfig config = new ProtobufDeserializerConfig(configs.originals(), isKey); super.configure(config, isKey); specificReturnClass = config.getSpecificReturnClass(); @@ -75,22 +69,21 @@ public void configure(Map configs, boolean isKey) { this.specificReturnClassParseMethod = specificReturnClass .getDeclaredMethod(PROTOBUF_PARSE_METHOD, InputStream.class); } else { - throw new ConfigException("Class " + specificReturnClass.getCanonicalName() + throw new IllegalStateException("Class " + specificReturnClass.getCanonicalName() + " is not a valid protobuf message class"); } } } catch (Exception e) { - throw new ConfigException("Class " + specificReturnClass.getCanonicalName() + throw new IllegalStateException("Class " + specificReturnClass.getCanonicalName() + " is not a valid protobuf message class", e); } deriveClass = config.deriveClass(); - serdeHeaders = new ProtobufSerdeHeaders(new HashMap<>(configs), isKey); } /** - * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + * @see io.apicurio.registry.serde.AbstractSerDe#schemaParser() */ @Override public SchemaParser schemaParser() { @@ -98,27 +91,17 @@ public SchemaParser schemaParser() { } /** - * @see io.apicurio.registry.serde.AbstractKafkaDeserializer#readData(org.apache.kafka.common.header.Headers, - * io.apicurio.registry.serde.ParsedSchema, java.nio.ByteBuffer, int, int) - */ - @Override - protected U readData(Headers headers, ParsedSchema schema, ByteBuffer buffer, int start, - int length) { - return internalReadData(headers, schema, buffer, start, length); - } - - /** - * @see io.apicurio.registry.serde.AbstractKafkaDeserializer#readData(io.apicurio.registry.serde.ParsedSchema, - * java.nio.ByteBuffer, int, int) + * @see AbstractDeserializer#readData(io.apicurio.registry.resolver.ParsedSchema, java.nio.ByteBuffer, + * int, int) */ @Override protected U readData(ParsedSchema schema, ByteBuffer buffer, int start, int length) { - return internalReadData(null, schema, buffer, start, length); + return internalReadData(schema, buffer, start, length); } @SuppressWarnings("unchecked") - protected U internalReadData(Headers headers, ParsedSchema schema, ByteBuffer buff, - int start, int length) { + protected U internalReadData(ParsedSchema schema, ByteBuffer buff, int start, + int length) { try { byte[] bytes = new byte[length]; System.arraycopy(buff.array(), start, bytes, 0, length); @@ -126,13 +109,13 @@ protected U internalReadData(Headers headers, ParsedSchema schem ByteArrayInputStream is = new ByteArrayInputStream(bytes); Descriptor descriptor = null; - if (headers != null) { - String messageTypeName = serdeHeaders.getProtobufTypeName(headers); - if (messageTypeName != null) { - descriptor = schema.getParsedSchema().getFileDescriptor() - .findMessageTypeByName(messageTypeName); - } + + if (messageTypeName != null) { + descriptor = schema.getParsedSchema().getFileDescriptor() + .findMessageTypeByName(messageTypeName); + } + if (descriptor == null) { try { Ref ref = Ref.parseDelimitedFrom(is); @@ -152,18 +135,15 @@ protected U internalReadData(Headers headers, ParsedSchema schem } return (U) specificReturnClassParseMethod.invoke(null, is); } catch (Exception e) { - throw new ConfigException("Not a valid protobuf builder", e); + throw new IllegalStateException("Not a valid protobuf builder", e); } } else if (deriveClass) { String className = deriveClassFromDescriptor(descriptor); if (className != null) { return invokeParseMethod(is, className); } - } else if (headers != null) { - String messageTypeHeader = serdeHeaders.getMessageType(headers); - if (messageTypeHeader != null) { - return invokeParseMethod(is, messageTypeHeader); - } + } else if (messageTypeName != null) { + return invokeParseMethod(is, messageTypeName); } return (U) DynamicMessage.parseFrom(descriptor, is); @@ -173,22 +153,26 @@ protected U internalReadData(Headers headers, ParsedSchema schem } } + public void setMessageTypeName(String messageTypeName) { + this.messageTypeName = messageTypeName; + } + @SuppressWarnings("unchecked") - private U invokeParseMethod(InputStream buffer, String className) { + public U invokeParseMethod(InputStream buffer, String className) { try { Method parseMethod = parseMethodsCache.computeIfAbsent(className, k -> { Class protobufClass = Utils.loadClass(className); try { return protobufClass.getDeclaredMethod(PROTOBUF_PARSE_METHOD, InputStream.class); } catch (NoSuchMethodException | SecurityException e) { - throw new SerializationException( + throw new IllegalStateException( "Class " + className + " is not a valid protobuf message class", e); } }); return (U) parseMethod.invoke(null, buffer); } catch (IllegalAccessException | InvocationTargetException e) { parseMethodsCache.remove(className); - throw new SerializationException("Not a valid protobuf builder", e); + throw new IllegalStateException("Not a valid protobuf builder", e); } } diff --git a/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufDeserializerConfig.java b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufDeserializerConfig.java new file mode 100644 index 0000000000..999a3b084e --- /dev/null +++ b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufDeserializerConfig.java @@ -0,0 +1,50 @@ +package io.apicurio.registry.serde.protobuf; + +import io.apicurio.registry.serde.config.SerdeConfig; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ProtobufDeserializerConfig extends SerdeConfig { + + public static final String SPECIFIC_RETURN_CLASS_DOC = "A class generated by Protocol buffers that the message value should be deserialized to"; + + public static final String DERIVE_CLASS_FROM_SCHEMA = "apicurio.protobuf.derive.class"; + public static final String DERIVE_CLASS_FROM_SCHEMA_DOC = "Whether to derive the class based on `java_outer_classname` and `java_multiple_files` from the Protobuf schema."; + + private boolean isKey; + + /** + * Constructor. + * + * @param originals + */ + public ProtobufDeserializerConfig(Map originals, boolean isKey) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + this.isKey = isKey; + } + + public Class getSpecificReturnClass() { + if (isKey) { + return this.getClass(DESERIALIZER_SPECIFIC_KEY_RETURN_CLASS); + } else { + return this.getClass(DESERIALIZER_SPECIFIC_VALUE_RETURN_CLASS); + } + } + + public boolean deriveClass() { + return this.getBoolean(DERIVE_CLASS_FROM_SCHEMA); + } + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } + + private static final Map DEFAULTS = Collections.emptyMap(); +} diff --git a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSchemaParser.java b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSchemaParser.java similarity index 96% rename from serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSchemaParser.java rename to serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSchemaParser.java index 579437fed6..2c9f349d1f 100644 --- a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSchemaParser.java +++ b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSchemaParser.java @@ -17,7 +17,6 @@ import io.apicurio.registry.utils.IoUtil; import io.apicurio.registry.utils.protobuf.schema.FileDescriptorUtils; import io.apicurio.registry.utils.protobuf.schema.ProtobufSchema; -import org.apache.kafka.common.errors.SerializationException; import java.util.ArrayList; import java.util.HashMap; @@ -27,7 +26,7 @@ public class ProtobufSchemaParser implements SchemaParser { /** - * @see io.apicurio.registry.serde.SchemaParser#artifactType() + * @see io.apicurio.registry.resolver.SchemaParser#artifactType() */ @Override public String artifactType() { @@ -35,7 +34,8 @@ public String artifactType() { } /** - * @see io.apicurio.registry.serde.SchemaParser#parseSchema(byte[]) + * @see io.apicurio.registry.resolver.SchemaParser#parseSchema(byte[], Map>) */ @Override public ProtobufSchema parseSchema(byte[] rawSchema, @@ -65,7 +65,7 @@ public ProtobufSchema parseSchema(byte[] rawSchema, return getFileDescriptorFromElement(fileElem); } } catch (DescriptorValidationException pe) { - throw new SerializationException("Error parsing protobuf schema ", pe); + throw new IllegalStateException("Error parsing protobuf schema ", pe); } catch (IllegalStateException illegalStateException) { // If qe get here the server likely returned the full descriptor, try to parse it. return parseDescriptor(rawSchema); diff --git a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializer.java b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerializer.java similarity index 61% rename from serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializer.java rename to serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerializer.java index 61a5bbab9a..26c0871b16 100644 --- a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializer.java +++ b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerializer.java @@ -8,56 +8,51 @@ import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.rules.compatibility.protobuf.ProtobufCompatibilityCheckerLibrary; -import io.apicurio.registry.serde.AbstractKafkaSerializer; +import io.apicurio.registry.serde.AbstractSerializer; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.protobuf.ref.RefOuterClass.Ref; import io.apicurio.registry.utils.protobuf.schema.ProtobufFile; import io.apicurio.registry.utils.protobuf.schema.ProtobufSchema; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.header.Headers; import java.io.IOException; import java.io.OutputStream; -import java.util.HashMap; import java.util.List; -import java.util.Map; -public class ProtobufKafkaSerializer extends AbstractKafkaSerializer { +public class ProtobufSerializer extends AbstractSerializer { private Boolean validationEnabled; - - private ProtobufSerdeHeaders serdeHeaders; private ProtobufSchemaParser parser = new ProtobufSchemaParser<>(); - public ProtobufKafkaSerializer() { + private boolean writeRef = true; + + public ProtobufSerializer() { super(); } - public ProtobufKafkaSerializer(RegistryClient client, + public ProtobufSerializer(RegistryClient client, ArtifactReferenceResolverStrategy artifactResolverStrategy, SchemaResolver schemaResolver) { super(client, artifactResolverStrategy, schemaResolver); } - public ProtobufKafkaSerializer(RegistryClient client) { + public ProtobufSerializer(RegistryClient client) { super(client); } - public ProtobufKafkaSerializer(SchemaResolver schemaResolver) { + public ProtobufSerializer(SchemaResolver schemaResolver) { super(schemaResolver); } @Override - public void configure(Map configs, boolean isKey) { - ProtobufKafkaSerializerConfig config = new ProtobufKafkaSerializerConfig(configs); + public void configure(SerdeConfig configs, boolean isKey) { + ProtobufSerializerConfig config = new ProtobufSerializerConfig(configs.originals()); super.configure(config, isKey); - serdeHeaders = new ProtobufSerdeHeaders(new HashMap<>(configs), isKey); - validationEnabled = config.validationEnabled(); } /** - * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + * @see io.apicurio.registry.serde.AbstractSerDe#schemaParser() */ @Override public SchemaParser schemaParser() { @@ -65,42 +60,28 @@ public SchemaParser schemaParser() { } /** - * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(io.apicurio.registry.serde.ParsedSchema, + * @see io.apicurio.registry.serde.AbstractSerializer#serializeData(io.apicurio.registry.resolver.ParsedSchema, * java.lang.Object, java.io.OutputStream) */ @Override protected void serializeData(ParsedSchema schema, U data, OutputStream out) throws IOException { - serializeData(null, schema, data, out); - } - - /** - * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(org.apache.kafka.common.header.Headers, - * io.apicurio.registry.serde.ParsedSchema, java.lang.Object, java.io.OutputStream) - */ - @Override - protected void serializeData(Headers headers, ParsedSchema schema, U data, - OutputStream out) throws IOException { if (validationEnabled) { if (schema.getParsedSchema() != null && schema.getParsedSchema().getFileDescriptor() .findMessageTypeByName(data.getDescriptorForType().getName()) == null) { - throw new SerializationException("Missing message type " + throw new IllegalStateException("Missing message type " + data.getDescriptorForType().getName() + " in the protobuf schema"); } List diffs = validate(schema, data); if (!diffs.isEmpty()) { - throw new SerializationException( + throw new IllegalStateException( "The data to send is not compatible with the schema. " + diffs); } } - - if (headers != null) { - serdeHeaders.addMessageTypeHeader(headers, data.getClass().getName()); - serdeHeaders.addProtobufTypeNameHeader(headers, data.getDescriptorForType().getName()); - } else { + if (writeRef) { Ref ref = Ref.newBuilder().setName(data.getDescriptorForType().getName()).build(); ref.writeDelimitedTo(out); } @@ -108,6 +89,10 @@ protected void serializeData(Headers headers, ParsedSchema schem data.writeTo(out); } + public void setWriteRef(boolean writeRef) { + this.writeRef = writeRef; + } + private List validate(ParsedSchema schemaFromRegistry, U data) { ProtobufFile fileBefore = schemaFromRegistry.getParsedSchema().getProtobufFile(); ProtobufFile fileAfter = new ProtobufFile( @@ -116,5 +101,4 @@ private List validate(ParsedSchema schemaFro fileAfter); return checker.findDifferences(); } - -} +} \ No newline at end of file diff --git a/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerializerConfig.java b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerializerConfig.java new file mode 100644 index 0000000000..52e3878756 --- /dev/null +++ b/serdes/generic/serde-common-protobuf/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerializerConfig.java @@ -0,0 +1,34 @@ +package io.apicurio.registry.serde.protobuf; + +import io.apicurio.registry.serde.config.SerdeConfig; + +import java.util.HashMap; +import java.util.Map; + +public class ProtobufSerializerConfig extends SerdeConfig { + + /** + * Constructor. + * + * @param originals + */ + public ProtobufSerializerConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + } + + public boolean validationEnabled() { + return this.getBoolean(VALIDATION_ENABLED); + } + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } + + private static final Map DEFAULTS = Map.of(VALIDATION_ENABLED, VALIDATION_ENABLED_DEFAULT); + +} diff --git a/serdes/protobuf-serde/src/main/proto/ref.proto b/serdes/generic/serde-common-protobuf/src/main/proto/ref.proto similarity index 100% rename from serdes/protobuf-serde/src/main/proto/ref.proto rename to serdes/generic/serde-common-protobuf/src/main/proto/ref.proto diff --git a/serdes/serde-common/pom.xml b/serdes/generic/serde-common/pom.xml similarity index 84% rename from serdes/serde-common/pom.xml rename to serdes/generic/serde-common/pom.xml index 1e42154c91..75c75c8ab0 100644 --- a/serdes/serde-common/pom.xml +++ b/serdes/generic/serde-common/pom.xml @@ -5,7 +5,7 @@ io.apicurio apicurio-registry 3.0.0-SNAPSHOT - ../../pom.xml + ../../../pom.xml apicurio-registry-serde-common @@ -13,7 +13,7 @@ apicurio-registry-serde-common - ${project.basedir}/../.. + ${project.basedir}/../../.. @@ -33,11 +33,6 @@ ${jboss-slf4j.version} - - org.apache.kafka - kafka-clients - - org.junit.jupiter junit-jupiter diff --git a/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractDeserializer.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractDeserializer.java new file mode 100644 index 0000000000..5ed20ba85f --- /dev/null +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractDeserializer.java @@ -0,0 +1,80 @@ +package io.apicurio.registry.serde; + +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaLookupResult; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReference; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.rest.client.RegistryClient; + +import java.nio.ByteBuffer; + +public abstract class AbstractDeserializer extends AbstractSerDe { + + public AbstractDeserializer() { + super(); + } + + public AbstractDeserializer(RegistryClient client) { + super(client); + } + + public AbstractDeserializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public AbstractDeserializer(RegistryClient client, SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public AbstractDeserializer(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + super(client, strategy, schemaResolver); + } + + public U deserializeData(String topic, byte[] data) { + if (data == null) { + return null; + } + + ByteBuffer buffer = getByteBuffer(data); + ArtifactReference artifactReference = getIdHandler().readId(buffer); + + SchemaLookupResult schema = resolve(topic, data, artifactReference); + + int length = buffer.limit() - 1 - getIdHandler().idSize(); + int start = buffer.position() + buffer.arrayOffset(); + + return readData(schema.getParsedSchema(), buffer, start, length); + } + + protected abstract U readData(ParsedSchema schema, ByteBuffer buffer, int start, int length); + + protected U readData(String topic, byte[] data, ArtifactReference artifactReference) { + SchemaLookupResult schema = resolve(topic, data, artifactReference); + + ByteBuffer buffer = ByteBuffer.wrap(data); + int length = buffer.limit(); + int start = buffer.position(); + + return readData(schema.getParsedSchema(), buffer, start, length); + } + + protected SchemaLookupResult resolve(String topic, byte[] data, ArtifactReference artifactReference) { + try { + return getSchemaResolver().resolveSchemaByArtifactReference(artifactReference); + } catch (RuntimeException e) { + if (fallbackArtifactProvider == null) { + throw e; + } else { + try { + ArtifactReference fallbackReference = fallbackArtifactProvider.get(topic, data); + return getSchemaResolver().resolveSchemaByArtifactReference(fallbackReference); + } catch (RuntimeException fe) { + fe.addSuppressed(e); + throw fe; + } + } + } + } +} diff --git a/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerDe.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerDe.java new file mode 100644 index 0000000000..bf2a20a928 --- /dev/null +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerDe.java @@ -0,0 +1,134 @@ +package io.apicurio.registry.serde; + +import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.resolver.utils.Utils; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.config.SerdeConfig; +import io.apicurio.registry.serde.config.SerdeDeserializerConfig; +import io.apicurio.registry.serde.fallback.DefaultFallbackArtifactProvider; +import io.apicurio.registry.serde.fallback.FallbackArtifactProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * Common class for both serializer and deserializer. + */ +public abstract class AbstractSerDe extends SchemaResolverConfigurer implements AutoCloseable { + + protected final Logger log = LoggerFactory.getLogger(getClass()); + + protected FallbackArtifactProvider fallbackArtifactProvider; + + public static final byte MAGIC_BYTE = 0x0; + + protected boolean key; // do we handle key or value with this ser/de? + + protected boolean isKey() { + return key; + } + + protected IdHandler idHandler; + + public AbstractSerDe() { + super(); + } + + public AbstractSerDe(RegistryClient client) { + super(client); + } + + public AbstractSerDe(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public AbstractSerDe(RegistryClient client, SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public AbstractSerDe(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + super(client, strategy, schemaResolver); + } + + public void configure(SerdeConfig config, boolean isKey) { + super.configure(config.originals(), isKey, schemaParser()); + + configureSerialization(config, isKey); + configureDeserialization(config, isKey); + } + + private void configureSerialization(SerdeConfig config, boolean isKey) { + // First we configure the serialization part. + key = isKey; + if (idHandler == null) { + Object idh = config.getIdHandler(); + Utils.instantiate(IdHandler.class, idh, this::setIdHandler); + } + idHandler.configure(config.originals(), isKey); + } + + private void configureDeserialization(SerdeConfig config, boolean isKey) { + SerdeDeserializerConfig deserializerConfig = new SerdeDeserializerConfig(config.originals()); + + Object fallbackProvider = deserializerConfig.getFallbackArtifactProvider(); + Utils.instantiate(FallbackArtifactProvider.class, fallbackProvider, + this::setFallbackArtifactProvider); + fallbackArtifactProvider.configure(config.originals(), isKey); + + if (fallbackArtifactProvider instanceof DefaultFallbackArtifactProvider) { + if (!((DefaultFallbackArtifactProvider) fallbackArtifactProvider).isConfigured()) { + // it's not configured, just remove it so it's not executed + fallbackArtifactProvider = null; + } + } + } + + public abstract SchemaParser schemaParser(); + + public IdHandler getIdHandler() { + return idHandler; + } + + public void setIdHandler(IdHandler idHandler) { + this.idHandler = Objects.requireNonNull(idHandler); + } + + /** + * @param fallbackArtifactProvider the fallbackArtifactProvider to set + */ + public void setFallbackArtifactProvider(FallbackArtifactProvider fallbackArtifactProvider) { + this.fallbackArtifactProvider = fallbackArtifactProvider; + } + + public void reset() { + schemaResolver.reset(); + } + + public static ByteBuffer getByteBuffer(byte[] payload) { + ByteBuffer buffer = ByteBuffer.wrap(payload); + if (buffer.get() != MAGIC_BYTE) { + throw new IllegalStateException("Unknown magic byte!"); + } + return buffer; + } + + @Override + public void close() { + try { + this.schemaResolver.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void as4ByteId() { + this.idHandler = new Default4ByteIdHandler(); + } +} \ No newline at end of file diff --git a/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerializer.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerializer.java new file mode 100644 index 0000000000..b242010eab --- /dev/null +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerializer.java @@ -0,0 +1,63 @@ +package io.apicurio.registry.serde; + +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaLookupResult; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.data.SerdeMetadata; +import io.apicurio.registry.serde.data.SerdeRecord; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; + +public abstract class AbstractSerializer extends AbstractSerDe { + + public AbstractSerializer() { + super(); + } + + public AbstractSerializer(RegistryClient client) { + super(client); + } + + public AbstractSerializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public AbstractSerializer(RegistryClient client, SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public AbstractSerializer(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + super(client, strategy, schemaResolver); + } + + protected abstract void serializeData(ParsedSchema schema, U data, OutputStream out) + throws IOException; + + public byte[] serializeData(String topic, U data) { + // just return null + if (data == null) { + return null; + } + try { + SerdeMetadata resolverMetadata = new SerdeMetadata(topic, isKey()); + + SchemaLookupResult schema = getSchemaResolver() + .resolveSchema(new SerdeRecord<>(resolverMetadata, data)); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(MAGIC_BYTE); + getIdHandler().writeId(schema.toArtifactReference(), out); + serializeData(schema.getParsedSchema(), data, out); + + return out.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/Default4ByteIdHandler.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/Default4ByteIdHandler.java similarity index 88% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/Default4ByteIdHandler.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/Default4ByteIdHandler.java index 661f186852..f50775819b 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/Default4ByteIdHandler.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/Default4ByteIdHandler.java @@ -1,9 +1,8 @@ package io.apicurio.registry.serde; import io.apicurio.registry.resolver.strategy.ArtifactReference; -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; import io.apicurio.registry.serde.config.IdOption; -import org.apache.kafka.common.errors.SerializationException; +import io.apicurio.registry.serde.config.SerdeConfig; import java.io.IOException; import java.io.OutputStream; @@ -23,7 +22,7 @@ public class Default4ByteIdHandler implements IdHandler { */ @Override public void configure(Map configs, boolean isKey) { - BaseKafkaSerDeConfig config = new BaseKafkaSerDeConfig(configs); + SerdeConfig config = new SerdeConfig(configs); idOption = config.useIdOption(); } @@ -32,7 +31,7 @@ public void writeId(ArtifactReference reference, OutputStream out) throws IOExce long id; if (idOption == IdOption.globalId) { if (reference.getGlobalId() == null) { - throw new SerializationException( + throw new IllegalStateException( "Missing globalId. IdOption is globalId but there is no contentId in the ArtifactReference"); } id = reference.getGlobalId(); @@ -47,7 +46,7 @@ public void writeId(ArtifactReference reference, ByteBuffer buffer) { long id; if (idOption == IdOption.globalId) { if (reference.getGlobalId() == null) { - throw new SerializationException( + throw new IllegalStateException( "Missing globalId. IdOption is globalId but there is no globalId in the ArtifactReference"); } id = reference.getGlobalId(); diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/IdHandler.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/IdHandler.java similarity index 100% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/IdHandler.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/IdHandler.java diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/Legacy8ByteIdHandler.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/Legacy8ByteIdHandler.java similarity index 84% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/Legacy8ByteIdHandler.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/Legacy8ByteIdHandler.java index eca77c246a..2bf8fc231c 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/Legacy8ByteIdHandler.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/Legacy8ByteIdHandler.java @@ -1,9 +1,8 @@ package io.apicurio.registry.serde; import io.apicurio.registry.resolver.strategy.ArtifactReference; -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; import io.apicurio.registry.serde.config.IdOption; -import org.apache.kafka.common.errors.SerializationException; +import io.apicurio.registry.serde.config.SerdeConfig; import java.io.IOException; import java.io.OutputStream; @@ -12,12 +11,11 @@ public class Legacy8ByteIdHandler implements IdHandler { static final int idSize = 8; // we use 8 / long - - private IdOption idOption = IdOption.contentId; + private IdOption idOption = IdOption.globalId; @Override public void configure(Map configs, boolean isKey) { - BaseKafkaSerDeConfig config = new BaseKafkaSerDeConfig(configs); + SerdeConfig config = new SerdeConfig(configs); idOption = config.useIdOption(); } @@ -26,7 +24,7 @@ public void writeId(ArtifactReference reference, OutputStream out) throws IOExce long id; if (idOption == IdOption.globalId) { if (reference.getGlobalId() == null) { - throw new SerializationException( + throw new IllegalStateException( "Missing globalId. IdOption is globalId but there is no contentId in the ArtifactReference"); } id = reference.getGlobalId(); @@ -41,7 +39,7 @@ public void writeId(ArtifactReference reference, ByteBuffer buffer) { long id; if (idOption == IdOption.globalId) { if (reference.getGlobalId() == null) { - throw new SerializationException( + throw new IllegalStateException( "Missing globalId. IdOption is globalId but there is no globalId in the ArtifactReference"); } id = reference.getGlobalId(); diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolverConfigurer.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolverConfigurer.java similarity index 83% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolverConfigurer.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolverConfigurer.java index e34ebbe948..96b6d04f36 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolverConfigurer.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolverConfigurer.java @@ -4,8 +4,10 @@ import io.apicurio.registry.resolver.SchemaParser; import io.apicurio.registry.resolver.SchemaResolver; import io.apicurio.registry.resolver.SchemaResolverConfig; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import io.apicurio.registry.resolver.utils.Utils; import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.config.SerdeConfig; import java.util.Map; import java.util.Objects; @@ -38,6 +40,14 @@ public SchemaResolverConfigurer(RegistryClient client, SchemaResolver sche getSchemaResolver().setClient(client); } + public SchemaResolverConfigurer(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + this(); + setSchemaResolver(schemaResolver); + getSchemaResolver().setClient(client); + getSchemaResolver().setArtifactResolverStrategy(strategy); + } + public SchemaResolver getSchemaResolver() { return schemaResolver; } @@ -66,5 +76,4 @@ protected void configure(Map configs, boolean isKey, SchemaParse configs.put(SerdeConfig.IS_KEY, isKey); getSchemaResolver().configure(configs, schemaParser); } - } \ No newline at end of file diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/IdOption.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/IdOption.java similarity index 100% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/IdOption.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/IdOption.java diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SerdeConfig.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/SerdeConfig.java similarity index 62% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/SerdeConfig.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/SerdeConfig.java index 33fc760afb..2c2367bd3e 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SerdeConfig.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/SerdeConfig.java @@ -1,23 +1,52 @@ -package io.apicurio.registry.serde; +package io.apicurio.registry.serde.config; import io.apicurio.registry.resolver.DefaultSchemaResolver; +import io.apicurio.registry.resolver.SchemaResolver; import io.apicurio.registry.resolver.SchemaResolverConfig; -import io.apicurio.registry.serde.config.IdOption; +import io.apicurio.registry.resolver.config.AbstractConfig; +import io.apicurio.registry.rest.v2.beans.IfExists; +import io.apicurio.registry.serde.Default4ByteIdHandler; +import io.apicurio.registry.serde.IdHandler; import io.apicurio.registry.serde.fallback.DefaultFallbackArtifactProvider; import io.apicurio.registry.serde.fallback.FallbackArtifactProvider; -import io.apicurio.registry.serde.headers.DefaultHeadersHandler; -import io.apicurio.registry.serde.headers.HeadersHandler; import io.apicurio.registry.serde.strategy.ArtifactResolverStrategy; import io.apicurio.registry.serde.strategy.TopicIdStrategy; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; +import static java.util.Map.entry; + /** * Contains all of the Serde configuration properties. These are all the property names used when configuring * serde classes in Kafka apps via a {@link Properties} object. Serde classes can be used by creating them * directly as well, in which case these property names are not relevant. */ -public class SerdeConfig { +public class SerdeConfig extends AbstractConfig { + + public SerdeConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + } + + public SerdeConfig() { + this.originals = DEFAULTS; + } + + public Object getIdHandler() { + return this.getObject(ID_HANDLER); + } + + public IdOption useIdOption() { + return IdOption.valueOf(this.getString(USE_ID)); + } + + @Override + protected Map getDefaults() { + return DEFAULTS; + } /** * Fully qualified Java classname of a class that implements {@link ArtifactResolverStrategy} and is @@ -48,6 +77,12 @@ public class SerdeConfig { public static final String AUTO_REGISTER_ARTIFACT = SchemaResolverConfig.AUTO_REGISTER_ARTIFACT; public static final boolean AUTO_REGISTER_ARTIFACT_DEFAULT = SchemaResolverConfig.AUTO_REGISTER_ARTIFACT_DEFAULT; + /** + * Used to indicate the auto-register feature to try to dereference the schema before registering it in + * Registry. Only supported for Avro. Only applicable when + * {@link SchemaResolverConfig#AUTO_REGISTER_ARTIFACT} is enabled. + */ + /** * Optional, one of {@link IfExists} to indicate the behavior of the client when there is a conflict * creating an artifact because the artifact already exists. @@ -55,6 +90,16 @@ public class SerdeConfig { public static final String AUTO_REGISTER_ARTIFACT_IF_EXISTS = SchemaResolverConfig.AUTO_REGISTER_ARTIFACT_IF_EXISTS; public static final String AUTO_REGISTER_ARTIFACT_IF_EXISTS_DEFAULT = SchemaResolverConfig.AUTO_REGISTER_ARTIFACT_IF_EXISTS_DEFAULT; + public static final String DEREFERENCE_SCHEMA = SchemaResolverConfig.DEREFERENCE_SCHEMA; + public static final boolean DEREFERENCE_SCHEMA_DEFAULT = SchemaResolverConfig.DEREFERENCE_SCHEMA_DEFAULT; + + /** + * Used to indicate the deserializer to ask Registry to return the schema dereferenced. This is useful to + * reduce the number of http requests to the server. Only applicable to Avro schemas. + */ + public static final String DESERIALIZER_DEREFERENCE_SCHEMA = SchemaResolverConfig.DESERIALIZER_DEREFERENCE_SCHEMA; + public static final boolean DESERIALIZER_DEREFERENCE_SCHEMA_DEFAULT = SchemaResolverConfig.DESERIALIZER_DEREFERENCE_SCHEMA_DEFAULT; + /** * Optional, boolean to indicate whether serializer classes should attempt to find the latest artifact in * the registry for the corresponding groupId/artifactId. GroupId and artifactId are configured either via @@ -133,27 +178,11 @@ public class SerdeConfig { /** * Fully qualified Java classname of a class that implements {@link IdHandler} and is responsible for - * writing the schema's Global ID to the message payload. Only used when - * {@link SerdeConfig#ENABLE_HEADERS} is missing or 'false'. + * writing the schema's Global ID to the message payload. */ public static final String ID_HANDLER = "apicurio.registry.id-handler"; public static final String ID_HANDLER_DEFAULT = Default4ByteIdHandler.class.getName(); - /** - * Boolean to indicate whether serde classes should pass Global Id information via message headers instead - * of in the message payload. - */ - public static final String ENABLE_HEADERS = "apicurio.registry.headers.enabled"; - public static final boolean ENABLE_HEADERS_DEFAULT = false; - - /** - * Fully qualified Java classname of a class that implements {@link HeadersHandler} and is responsible for - * writing the schema's Global ID to the message headers. Only used when - * {@link SerdeConfig#ENABLE_HEADERS} is 'true'. - */ - public static final String HEADERS_HANDLER = "apicurio.registry.headers.handler"; - public static final String HEADERS_HANDLER_DEFAULT = DefaultHeadersHandler.class.getName(); - /** * Indicates how long to cache artifacts before auto-eviction. If not included, the artifact will be * fetched every time. @@ -235,89 +264,7 @@ public class SerdeConfig { */ public static final String DESERIALIZER_SPECIFIC_VALUE_RETURN_CLASS = "apicurio.registry.deserializer.value.return-class"; - /** - * Used to override the Kafka message header name used to pass the groupId for the message key. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_KEY_GROUP_ID}. - */ - public static final String HEADER_KEY_GROUP_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.groupId.name"; - /** - * Used to override the Kafka message header name used to pass the groupId for the message value. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_VALUE_GROUP_ID}. - */ - public static final String HEADER_VALUE_GROUP_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.groupId.name"; - /** - * Used to override the Kafka message header name used to pass the artifactId for the message key. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_KEY_ARTIFACT_ID}. - */ - public static final String HEADER_KEY_ARTIFACT_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.artifactId.name"; - /** - * Used to override the Kafka message header name used to pass the artifactId for the message value. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_VALUE_ARTIFACT_ID}. - */ - public static final String HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.artifactId.name"; - /** - * Used to override the Kafka message header name used to pass the version for the message key. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_KEY_VERSION}. - */ - public static final String HEADER_KEY_VERSION_OVERRIDE_NAME = "apicurio.registry.headers.key.version.name"; - /** - * Used to override the Kafka message header name used to pass the version for the message value. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_VALUE_VERSION}. - */ - public static final String HEADER_VALUE_VERSION_OVERRIDE_NAME = "apicurio.registry.headers.value.version.name"; - /** - * Used to override the Kafka message header name used to pass the globalId for the message key. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_KEY_GLOBAL_ID}. - */ - public static final String HEADER_KEY_GLOBAL_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.globalId.name"; - /** - * Used to override the Kafka message header name used to pass the globalId for the message value. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_VALUE_GLOBAL_ID}. - */ - public static final String HEADER_VALUE_GLOBAL_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.globalId.name"; - /** - * Used to override the Kafka message header name used to pass the contentId for the message key. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_KEY_CONTENT_ID}. - */ - public static final String HEADER_KEY_CONTENT_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.contentId.name"; - /** - * Used to override the Kafka message header name used to pass the contentId for the message value. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_VALUE_CONTENT_ID}. - */ - public static final String HEADER_VALUE_CONTENT_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.contentId.name"; - /** - * Used to override the Kafka message header name used to pass the contentHash for the message key. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_KEY_CONTENT_HASH}. - */ - public static final String HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME = "apicurio.registry.headers.key.contentHash.name"; - /** - * Used to override the Kafka message header name used to pass the contentHash for the message value. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Default value is - * {@link SerdeHeaders#HEADER_VALUE_CONTENT_HASH}. - */ - public static final String HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME = "apicurio.registry.headers.value.contentHash.name"; - /** - * Used to override the Kafka message header name used to pass the message type for the message key. Only - * applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Only used by the JSON Schema serde - * classes. Default value is {@link SerdeHeaders#HEADER_KEY_MESSAGE_TYPE}. - */ - public static final String HEADER_KEY_MESSAGE_TYPE_OVERRIDE_NAME = "apicurio.registry.headers.key.msgType.name"; - /** - * Used to override the Kafka message header name used to pass the message type for the message value. - * Only applicable when {@link SerdeConfig#ENABLE_HEADERS} is enabled. Only used by the JSON Schema serde - * classes. Default value is {@link SerdeHeaders#HEADER_VALUE_MESSAGE_TYPE}. - */ - public static final String HEADER_VALUE_MESSAGE_TYPE_OVERRIDE_NAME = "apicurio.registry.headers.value.msgType.name"; + private static final Map DEFAULTS = Map.ofEntries(entry(ID_HANDLER, ID_HANDLER_DEFAULT), + entry(USE_ID, USE_ID_DEFAULT)); } diff --git a/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/SerdeDeserializerConfig.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/SerdeDeserializerConfig.java new file mode 100644 index 0000000000..62d9cd6ab7 --- /dev/null +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/config/SerdeDeserializerConfig.java @@ -0,0 +1,31 @@ +package io.apicurio.registry.serde.config; + +import java.util.HashMap; +import java.util.Map; + +import static io.apicurio.registry.serde.config.SerdeConfig.FALLBACK_ARTIFACT_PROVIDER; +import static io.apicurio.registry.serde.config.SerdeConfig.FALLBACK_ARTIFACT_PROVIDER_DEFAULT; +import static java.util.Map.entry; + +public class SerdeDeserializerConfig extends SerdeConfig { + + public SerdeDeserializerConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + } + + private static final Map DEFAULTS = Map + .ofEntries(entry(FALLBACK_ARTIFACT_PROVIDER, FALLBACK_ARTIFACT_PROVIDER_DEFAULT)); + + public Object getFallbackArtifactProvider() { + return this.getObject(FALLBACK_ARTIFACT_PROVIDER); + } + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } +} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeMetadata.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/data/SerdeMetadata.java similarity index 64% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeMetadata.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/data/SerdeMetadata.java index ab350dfcc8..38fc9a6a5b 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeMetadata.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/data/SerdeMetadata.java @@ -2,21 +2,18 @@ import io.apicurio.registry.resolver.data.Metadata; import io.apicurio.registry.resolver.strategy.ArtifactReference; -import org.apache.kafka.common.header.Headers; /** * Kafka specific implementation for the Record Metadata abstraction used by the SchemaResolver */ -public class KafkaSerdeMetadata implements Metadata { +public class SerdeMetadata implements Metadata { - private String topic; - private boolean isKey; - private Headers headers; + private final String topic; + private final boolean isKey; - public KafkaSerdeMetadata(String topic, boolean isKey, Headers headers) { + public SerdeMetadata(String topic, boolean isKey) { this.topic = topic; this.isKey = isKey; - this.headers = headers; } /** @@ -41,11 +38,4 @@ public boolean isKey() { return isKey; } - /** - * @return the headers - */ - public Headers getHeaders() { - return headers; - } - } diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeRecord.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/data/SerdeRecord.java similarity index 65% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeRecord.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/data/SerdeRecord.java index b043be5508..d5f452c08e 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeRecord.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/data/SerdeRecord.java @@ -2,12 +2,12 @@ import io.apicurio.registry.resolver.data.Record; -public class KafkaSerdeRecord implements Record { +public class SerdeRecord implements Record { - private KafkaSerdeMetadata metadata; - private T payload; + private final SerdeMetadata metadata; + private final T payload; - public KafkaSerdeRecord(KafkaSerdeMetadata metadata, T payload) { + public SerdeRecord(SerdeMetadata metadata, T payload) { this.metadata = metadata; this.payload = payload; } @@ -16,7 +16,7 @@ public KafkaSerdeRecord(KafkaSerdeMetadata metadata, T payload) { * @see io.apicurio.registry.resolver.data.Record#metadata() */ @Override - public KafkaSerdeMetadata metadata() { + public SerdeMetadata metadata() { return metadata; } diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/fallback/DefaultFallbackArtifactProvider.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/fallback/DefaultFallbackArtifactProvider.java similarity index 81% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/fallback/DefaultFallbackArtifactProvider.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/fallback/DefaultFallbackArtifactProvider.java index 071d5b674f..7cf7b52c7e 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/fallback/DefaultFallbackArtifactProvider.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/fallback/DefaultFallbackArtifactProvider.java @@ -1,8 +1,7 @@ package io.apicurio.registry.serde.fallback; import io.apicurio.registry.resolver.strategy.ArtifactReference; -import io.apicurio.registry.serde.SerdeConfig; -import org.apache.kafka.common.header.Headers; +import io.apicurio.registry.serde.config.SerdeConfig; import java.util.Map; @@ -14,7 +13,7 @@ public class DefaultFallbackArtifactProvider implements FallbackArtifactProvider private ArtifactReference fallbackArtifactReference; /** - * @see io.apicurio.registry.serde.FallbackArtifactProvider#configure(java.util.Map, boolean) + * @see io.apicurio.registry.serde.fallback.FallbackArtifactProvider#configure(java.util.Map, boolean) */ @Override public void configure(Map configs, boolean isKey) { @@ -45,11 +44,10 @@ public void configure(Map configs, boolean isKey) { } /** - * @see io.apicurio.registry.serde.fallback.FallbackArtifactProvider#get(java.lang.String, - * org.apache.kafka.common.header.Headers, byte[]) + * @see io.apicurio.registry.serde.fallback.FallbackArtifactProvider#get(java.lang.String, byte[]) */ @Override - public ArtifactReference get(String topic, Headers headers, byte[] data) { + public ArtifactReference get(String topic, byte[] data) { return fallbackArtifactReference; } diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/fallback/FallbackArtifactProvider.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/fallback/FallbackArtifactProvider.java similarity index 80% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/fallback/FallbackArtifactProvider.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/fallback/FallbackArtifactProvider.java index b65c5daf1a..604d389db0 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/fallback/FallbackArtifactProvider.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/fallback/FallbackArtifactProvider.java @@ -1,7 +1,6 @@ package io.apicurio.registry.serde.fallback; import io.apicurio.registry.resolver.strategy.ArtifactReference; -import org.apache.kafka.common.header.Headers; import java.util.Map; @@ -19,10 +18,9 @@ default void configure(Map configs, boolean isKey) { * artifact that will be used to deserialize the kafka message * * @param topic - * @param headers , can be null * @param data * @return */ - public ArtifactReference get(String topic, Headers headers, byte[] data); + public ArtifactReference get(String topic, byte[] data); } diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactReference.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactReference.java similarity index 95% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactReference.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactReference.java index fca5fcb974..6d6d8f1099 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactReference.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactReference.java @@ -5,7 +5,7 @@ /** * There is a new implementation of this class that can be found here * {@link io.apicurio.registry.resolver.strategy.ArtifactReferenceImpl} and here - * {@linkio.apicurio.registry.resolver.strategy.ArtifactReference} We keep this class for compatibilty This + * {@link io.apicurio.registry.resolver.strategy.ArtifactReference} We keep this class for compatibilty This * class holds the information that reference one Artifact in Apicurio Registry. It will always make reference * to an artifact in a group. Optionally it can reference to a specific version. */ diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactResolverStrategy.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactResolverStrategy.java similarity index 88% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactResolverStrategy.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactResolverStrategy.java index 5919adb063..ae2afaf9cb 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactResolverStrategy.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/ArtifactResolverStrategy.java @@ -3,8 +3,8 @@ import io.apicurio.registry.resolver.ParsedSchema; import io.apicurio.registry.resolver.data.Record; import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; -import io.apicurio.registry.serde.data.KafkaSerdeMetadata; -import io.apicurio.registry.serde.data.KafkaSerdeRecord; +import io.apicurio.registry.serde.data.SerdeMetadata; +import io.apicurio.registry.serde.data.SerdeRecord; /** * There is a new interface responsible with the same responsibility as this one, can be found here @@ -31,8 +31,8 @@ public interface ArtifactResolverStrategy extends ArtifactReferenceResolverSt @Override default io.apicurio.registry.resolver.strategy.ArtifactReference artifactReference(Record data, ParsedSchema parsedSchema) { - KafkaSerdeRecord kdata = (KafkaSerdeRecord) data; - KafkaSerdeMetadata metadata = kdata.metadata(); + SerdeRecord kdata = (SerdeRecord) data; + SerdeMetadata metadata = kdata.metadata(); return artifactReference(metadata.getTopic(), metadata.isKey(), parsedSchema == null ? null : parsedSchema.getParsedSchema()); } diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/SimpleTopicIdStrategy.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/SimpleTopicIdStrategy.java similarity index 88% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/SimpleTopicIdStrategy.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/SimpleTopicIdStrategy.java index 63894e5f7f..e39b03372a 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/SimpleTopicIdStrategy.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/SimpleTopicIdStrategy.java @@ -4,7 +4,7 @@ import io.apicurio.registry.resolver.data.Record; import io.apicurio.registry.resolver.strategy.ArtifactReference; import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; -import io.apicurio.registry.serde.data.KafkaSerdeRecord; +import io.apicurio.registry.serde.data.SerdeRecord; public class SimpleTopicIdStrategy implements ArtifactReferenceResolverStrategy { @@ -14,7 +14,7 @@ public class SimpleTopicIdStrategy implements ArtifactReferenceResolverStrate */ @Override public ArtifactReference artifactReference(Record data, ParsedSchema parsedSchema) { - KafkaSerdeRecord kdata = (KafkaSerdeRecord) data; + SerdeRecord kdata = (SerdeRecord) data; return ArtifactReference.builder().groupId(null).artifactId(kdata.metadata().getTopic()).build(); } diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/TopicIdStrategy.java b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/TopicIdStrategy.java similarity index 78% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/TopicIdStrategy.java rename to serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/TopicIdStrategy.java index 09fe5e7679..dc999e587d 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/strategy/TopicIdStrategy.java +++ b/serdes/generic/serde-common/src/main/java/io/apicurio/registry/serde/strategy/TopicIdStrategy.java @@ -4,15 +4,15 @@ import io.apicurio.registry.resolver.data.Record; import io.apicurio.registry.resolver.strategy.ArtifactReference; import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; -import io.apicurio.registry.serde.data.KafkaSerdeMetadata; -import io.apicurio.registry.serde.data.KafkaSerdeRecord; +import io.apicurio.registry.serde.data.SerdeMetadata; +import io.apicurio.registry.serde.data.SerdeRecord; public class TopicIdStrategy implements ArtifactReferenceResolverStrategy { @Override public ArtifactReference artifactReference(Record data, ParsedSchema parsedSchema) { - KafkaSerdeRecord kdata = (KafkaSerdeRecord) data; - KafkaSerdeMetadata metadata = kdata.metadata(); + SerdeRecord kdata = (SerdeRecord) data; + SerdeMetadata metadata = kdata.metadata(); return ArtifactReference.builder().groupId(null) .artifactId(String.format("%s-%s", metadata.getTopic(), metadata.isKey() ? "key" : "value")) .build(); diff --git a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializerConfig.java b/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializerConfig.java deleted file mode 100644 index 777e7ddfca..0000000000 --- a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializerConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -package io.apicurio.registry.serde.jsonschema; - -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; - -import java.util.Map; - -import static io.apicurio.registry.serde.SerdeConfig.*; - -public class JsonSchemaKafkaDeserializerConfig extends BaseKafkaSerDeConfig { - - public static final String SPECIFIC_RETURN_CLASS_DOC = "The specific class to use for deserializing the data into java objects"; - - private static ConfigDef configDef() { - ConfigDef configDef = new ConfigDef() - .define(DESERIALIZER_SPECIFIC_KEY_RETURN_CLASS, Type.CLASS, null, Importance.MEDIUM, - SPECIFIC_RETURN_CLASS_DOC) - .define(DESERIALIZER_SPECIFIC_VALUE_RETURN_CLASS, Type.CLASS, null, Importance.MEDIUM, - SPECIFIC_RETURN_CLASS_DOC) - .define(VALIDATION_ENABLED, Type.BOOLEAN, VALIDATION_ENABLED_DEFAULT, Importance.MEDIUM, - "Whether to validate the data against the json schema"); - return configDef; - } - - private boolean isKey; - - /** - * Constructor. - * - * @param originals - */ - public JsonSchemaKafkaDeserializerConfig(Map originals, boolean isKey) { - super(configDef(), originals); - this.isKey = isKey; - - } - - public Class getSpecificReturnClass() { - if (isKey) { - return this.getClass(DESERIALIZER_SPECIFIC_KEY_RETURN_CLASS); - } else { - return this.getClass(DESERIALIZER_SPECIFIC_VALUE_RETURN_CLASS); - } - } - - public boolean validationEnabled() { - return this.getBoolean(VALIDATION_ENABLED); - } -} diff --git a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializerConfig.java b/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializerConfig.java deleted file mode 100644 index 03c4666737..0000000000 --- a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializerConfig.java +++ /dev/null @@ -1,32 +0,0 @@ -package io.apicurio.registry.serde.jsonschema; - -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; - -import java.util.Map; - -import static io.apicurio.registry.serde.SerdeConfig.*; - -public class JsonSchemaKafkaSerializerConfig extends BaseKafkaSerDeConfig { - - private static ConfigDef configDef() { - return new ConfigDef().define(VALIDATION_ENABLED, Type.BOOLEAN, VALIDATION_ENABLED_DEFAULT, - Importance.MEDIUM, "Whether to validate the data against the json schema"); - } - - /** - * Constructor. - * - * @param originals - */ - public JsonSchemaKafkaSerializerConfig(Map originals) { - super(configDef(), originals); - - } - - public boolean validationEnabled() { - return this.getBoolean(VALIDATION_ENABLED); - } -} diff --git a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerde.java b/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerde.java deleted file mode 100644 index 7a3b02238f..0000000000 --- a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerde.java +++ /dev/null @@ -1,12 +0,0 @@ -package io.apicurio.registry.serde.jsonschema; - -import io.apicurio.registry.serde.AbstractSerde; - -/** - * Wraps the JsonSchemaKafkaSerializer and JsonSchemaKafkaDeserializer. - */ -public class JsonSchemaSerde extends AbstractSerde { - public JsonSchemaSerde() { - super(new JsonSchemaKafkaSerializer<>(), new JsonSchemaKafkaDeserializer<>()); - } -} diff --git a/serdes/kafka/avro-serde/pom.xml b/serdes/kafka/avro-serde/pom.xml new file mode 100644 index 0000000000..077dc1eea1 --- /dev/null +++ b/serdes/kafka/avro-serde/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + io.apicurio + apicurio-registry + 3.0.0-SNAPSHOT + ../../../pom.xml + + + apicurio-registry-avro-serde-kafka + jar + apicurio-registry-avro-serde-kafka + + + ${project.basedir}/../../.. + + + + + io.apicurio + apicurio-registry-serde-kafka-common + + + + io.apicurio + apicurio-registry-serde-common-avro + + + + + diff --git a/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaDeserializer.java b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaDeserializer.java new file mode 100644 index 0000000000..3c37651125 --- /dev/null +++ b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaDeserializer.java @@ -0,0 +1,68 @@ +package io.apicurio.registry.serde.avro; + +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.AbstractKafkaDeserializer; +import org.apache.avro.Schema; +import org.apache.kafka.common.header.Headers; + +import java.nio.ByteBuffer; +import java.util.Map; + +public class AvroKafkaDeserializer extends AbstractKafkaDeserializer { + + private AvroSerdeHeaders avroHeaders; + private AvroDeserializer avroDeserializer; + + public AvroKafkaDeserializer() { + super(); + } + + public AvroKafkaDeserializer(RegistryClient client) { + super(client); + } + + @SuppressWarnings("rawtypes") + @Override + public void configure(Map configs, boolean isKey) { + avroHeaders = new AvroSerdeHeaders(isKey); + AvroSerdeConfig avroSerdeConfig = new AvroSerdeConfig(configs); + this.avroDeserializer = new AvroDeserializer<>(); + if (getSchemaResolver() != null) { + this.avroDeserializer.setSchemaResolver(getSchemaResolver()); + } + avroDeserializer.configure(avroSerdeConfig, isKey); + + super.configure(configs, isKey); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + */ + @Override + public SchemaParser schemaParser() { + return avroDeserializer.schemaParser(); + } + + @Override + protected U readData(ParsedSchema schema, ByteBuffer buffer, int start, int length) { + return avroDeserializer.readData(schema, buffer, start, length); + } + + @Override + public U deserialize(String topic, Headers headers, byte[] data) { + AvroEncoding encoding = null; + if (headers != null) { + String encodingHeader = avroHeaders.getEncoding(headers); + if (encodingHeader != null) { + encoding = AvroEncoding.valueOf(encodingHeader); + } + } + if (encoding != null) { + avroDeserializer.setEncoding(encoding); + } + + return super.deserialize(topic, headers, data); + } +} diff --git a/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerializer.java b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerializer.java new file mode 100644 index 0000000000..6ede993226 --- /dev/null +++ b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroKafkaSerializer.java @@ -0,0 +1,84 @@ +package io.apicurio.registry.serde.avro; + +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.AbstractKafkaSerializer; +import org.apache.avro.Schema; +import org.apache.kafka.common.header.Headers; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +public class AvroKafkaSerializer extends AbstractKafkaSerializer { + + AvroSerializer avroSerializer; + + private AvroSerdeHeaders avroHeaders; + + public AvroKafkaSerializer() { + super(); + } + + public AvroKafkaSerializer(RegistryClient client) { + super(client); + } + + public AvroKafkaSerializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public AvroKafkaSerializer(RegistryClient client, + ArtifactReferenceResolverStrategy artifactResolverStrategy, + SchemaResolver schemaResolver) { + super(client, artifactResolverStrategy, schemaResolver); + } + + @SuppressWarnings("rawtypes") + @Override + public void configure(Map configs, boolean isKey) { + avroHeaders = new AvroSerdeHeaders(isKey); + AvroSerdeConfig avroSerdeConfig = new AvroSerdeConfig(configs); + this.avroSerializer = new AvroSerializer<>(); + if (getSchemaResolver() != null) { + this.avroSerializer.setSchemaResolver(getSchemaResolver()); + } + avroSerializer.configure(avroSerdeConfig, isKey); + + super.configure(configs, isKey); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + */ + @Override + public SchemaParser schemaParser() { + return avroSerializer.schemaParser(); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(io.apicurio.registry.resolver.ParsedSchema, + * java.lang.Object, java.io.OutputStream) + */ + @SuppressWarnings("unchecked") + @Override + protected void serializeData(ParsedSchema schema, U data, OutputStream out) throws IOException { + avroSerializer.serializeData(schema, data, out); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(org.apache.kafka.common.header.Headers, + * io.apicurio.registry.resolver.ParsedSchema, java.lang.Object, java.io.OutputStream) + */ + @Override + protected void serializeData(Headers headers, ParsedSchema schema, U data, OutputStream out) + throws IOException { + if (headers != null) { + avroHeaders.addEncodingHeader(headers, avroSerializer.getEncoding().name()); + } + serializeData(schema, data, out); + } +} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerde.java b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerde.java similarity index 77% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerde.java rename to serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerde.java index 79c00f83bc..665e7bc44c 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSerde.java +++ b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerde.java @@ -1,4 +1,4 @@ -package io.apicurio.registry.serde; +package io.apicurio.registry.serde.avro; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; @@ -6,11 +6,15 @@ import java.util.Map; -public abstract class AbstractSerde implements Serde { +/** + * Wraps the AvroKafkaSerializer and AvroKafkaDeserializer. + */ +public class AvroSerde implements Serde { + final private Serializer serializer; final private Deserializer deserializer; - protected AbstractSerde(Serializer serializer, Deserializer deserializer) { + protected AvroSerde(Serializer serializer, Deserializer deserializer) { this.serializer = serializer; this.deserializer = deserializer; } diff --git a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeHeaders.java b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeHeaders.java similarity index 80% rename from serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeHeaders.java rename to serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeHeaders.java index 82c2661d5e..6dab8a62b1 100644 --- a/serdes/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeHeaders.java +++ b/serdes/kafka/avro-serde/src/main/java/io/apicurio/registry/serde/avro/AvroSerdeHeaders.java @@ -1,6 +1,6 @@ package io.apicurio.registry.serde.avro; -import io.apicurio.registry.serde.SerdeHeaders; +import io.apicurio.registry.serde.headers.KafkaSerdeHeaders; import io.apicurio.registry.utils.IoUtil; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; @@ -12,9 +12,9 @@ public class AvroSerdeHeaders { public AvroSerdeHeaders(boolean isKey) { if (isKey) { - encodingHeaderName = SerdeHeaders.HEADER_KEY_ENCODING; + encodingHeaderName = KafkaSerdeHeaders.HEADER_KEY_ENCODING; } else { - encodingHeaderName = SerdeHeaders.HEADER_VALUE_ENCODING; + encodingHeaderName = KafkaSerdeHeaders.HEADER_VALUE_ENCODING; } } diff --git a/serdes/kafka/jsonschema-serde/pom.xml b/serdes/kafka/jsonschema-serde/pom.xml new file mode 100644 index 0000000000..906442e6c8 --- /dev/null +++ b/serdes/kafka/jsonschema-serde/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + io.apicurio + apicurio-registry + 3.0.0-SNAPSHOT + ../../../pom.xml + + + apicurio-registry-jsonschema-serde-kafka + jar + apicurio-registry-jsonschema-serde-kafka + + + ${project.basedir}/../../.. + + + + + io.apicurio + apicurio-registry-serde-kafka-common + + + + io.apicurio + apicurio-registry-serde-common-jsonschema + + + + + diff --git a/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java new file mode 100644 index 0000000000..b162c42166 --- /dev/null +++ b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaDeserializer.java @@ -0,0 +1,82 @@ +package io.apicurio.registry.serde.jsonschema; + +import com.networknt.schema.JsonSchema; +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.utils.Utils; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.AbstractKafkaDeserializer; +import io.apicurio.registry.serde.headers.MessageTypeSerdeHeaders; +import org.apache.kafka.common.header.Headers; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +public class JsonSchemaKafkaDeserializer extends AbstractKafkaDeserializer { + + private MessageTypeSerdeHeaders serdeHeaders; + private JsonSchemaDeserializer jsonSchemaDeserializer; + + public JsonSchemaKafkaDeserializer() { + super(); + } + + public JsonSchemaKafkaDeserializer(RegistryClient client, SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public JsonSchemaKafkaDeserializer(RegistryClient client) { + super(client); + } + + public JsonSchemaKafkaDeserializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + /** + * @see AbstractKafkaDeserializer#configure(java.util.Map, boolean) + */ + @SuppressWarnings("unchecked") + @Override + public void configure(Map configs, boolean isKey) { + JsonSchemaDeserializerConfig config = new JsonSchemaDeserializerConfig(configs, isKey); + this.jsonSchemaDeserializer = new JsonSchemaDeserializer<>(); + if (getSchemaResolver() != null) { + this.jsonSchemaDeserializer.setSchemaResolver(getSchemaResolver()); + } + jsonSchemaDeserializer.configure(config, isKey); + + this.serdeHeaders = new MessageTypeSerdeHeaders(new HashMap<>(configs), isKey); + super.configure(configs, isKey); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + */ + @Override + public SchemaParser schemaParser() { + return jsonSchemaDeserializer.schemaParser(); + } + + /** + * @see AbstractKafkaDeserializer#readData(io.apicurio.registry.resolver.ParsedSchema, + * java.nio.ByteBuffer, int, int) + */ + @Override + protected T readData(ParsedSchema schema, ByteBuffer buffer, int start, int length) { + return jsonSchemaDeserializer.readData(schema, buffer, start, length); + } + + @Override + public T deserialize(String topic, Headers headers, byte[] data) { + if (headers != null && jsonSchemaDeserializer.getSpecificReturnClass() == null) { + String javaType = serdeHeaders.getMessageType(headers); + jsonSchemaDeserializer + .setSpecificReturnClass(javaType == null ? null : Utils.loadClass(javaType)); + } + + return super.deserialize(topic, headers, data); + } +} diff --git a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializer.java b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializer.java similarity index 62% rename from serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializer.java rename to serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializer.java index 06ab2b7cff..9991bb1707 100644 --- a/serdes/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializer.java +++ b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaKafkaSerializer.java @@ -1,8 +1,5 @@ package io.apicurio.registry.serde.jsonschema; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.networknt.schema.JsonSchema; import io.apicurio.registry.resolver.ParsedSchema; import io.apicurio.registry.resolver.SchemaParser; @@ -12,7 +9,6 @@ import io.apicurio.registry.serde.AbstractKafkaSerializer; import io.apicurio.registry.serde.headers.MessageTypeSerdeHeaders; import org.apache.kafka.common.header.Headers; -import org.apache.kafka.common.serialization.Serializer; import java.io.IOException; import java.io.OutputStream; @@ -24,15 +20,12 @@ * user's application needs to serialize a Java Bean to JSON data using Jackson. In addition to standard * serialization of the bean, this implementation can also optionally validate it against a JSON schema. */ -public class JsonSchemaKafkaSerializer extends AbstractKafkaSerializer - implements Serializer { +public class JsonSchemaKafkaSerializer extends AbstractKafkaSerializer { - private ObjectMapper mapper; - private final JsonSchemaParser parser = new JsonSchemaParser<>(); - - private Boolean validationEnabled; private MessageTypeSerdeHeaders serdeHeaders; + private JsonSchemaSerializer jsonSchemaSerializer; + public JsonSchemaKafkaSerializer() { super(); } @@ -51,45 +44,27 @@ public JsonSchemaKafkaSerializer(SchemaResolver schemaResolver) { super(schemaResolver); } - public JsonSchemaKafkaSerializer(RegistryClient client, Boolean validationEnabled) { - this(client); - this.validationEnabled = validationEnabled; - } - /** * @see io.apicurio.registry.serde.AbstractKafkaSerializer#configure(java.util.Map, boolean) */ @Override public void configure(Map configs, boolean isKey) { - JsonSchemaKafkaSerializerConfig config = new JsonSchemaKafkaSerializerConfig(configs); - super.configure(config, isKey); - - if (validationEnabled == null) { - this.validationEnabled = config.validationEnabled(); + JsonSchemaSerializerConfig config = new JsonSchemaSerializerConfig(configs); + this.jsonSchemaSerializer = new JsonSchemaSerializer<>(); + if (getSchemaResolver() != null) { + this.jsonSchemaSerializer.setSchemaResolver(getSchemaResolver()); } - + jsonSchemaSerializer.configure(config, isKey); serdeHeaders = new MessageTypeSerdeHeaders(new HashMap<>(configs), isKey); - if (null == mapper) { - this.mapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL); - } - } - - public boolean isValidationEnabled() { - return validationEnabled != null && validationEnabled; - } - - public void setObjectMapper(ObjectMapper objectMapper) { - this.mapper = objectMapper; + super.configure(configs, isKey); } /** * @param validationEnabled the validationEnabled to set */ public void setValidationEnabled(Boolean validationEnabled) { - this.validationEnabled = validationEnabled; + this.jsonSchemaSerializer.setValidationEnabled(validationEnabled); } /** @@ -97,33 +72,31 @@ public void setValidationEnabled(Boolean validationEnabled) { */ @Override public SchemaParser schemaParser() { - return parser; + return jsonSchemaSerializer.schemaParser(); } /** - * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(io.apicurio.registry.serde.ParsedSchema, + * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(io.apicurio.registry.resolver.ParsedSchema, * java.lang.Object, java.io.OutputStream) */ @Override protected void serializeData(ParsedSchema schema, T data, OutputStream out) throws IOException { - serializeData(null, schema, data, out); + jsonSchemaSerializer.serializeData(schema, data, out); } /** * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(org.apache.kafka.common.header.Headers, - * io.apicurio.registry.serde.ParsedSchema, java.lang.Object, java.io.OutputStream) + * io.apicurio.registry.resolver.ParsedSchema, java.lang.Object, java.io.OutputStream) */ @Override protected void serializeData(Headers headers, ParsedSchema schema, T data, OutputStream out) throws IOException { - final byte[] dataBytes = mapper.writeValueAsBytes(data); - if (isValidationEnabled()) { - JsonSchemaValidationUtil.validateDataWithSchema(schema, dataBytes, mapper); - } + if (headers != null) { serdeHeaders.addMessageTypeHeader(headers, data.getClass().getName()); } - out.write(dataBytes); + + serializeData(schema, data, out); } } diff --git a/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerde.java b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerde.java new file mode 100644 index 0000000000..5d4426dfeb --- /dev/null +++ b/serdes/kafka/jsonschema-serde/src/main/java/io/apicurio/registry/serde/jsonschema/JsonSchemaSerde.java @@ -0,0 +1,43 @@ +package io.apicurio.registry.serde.jsonschema; + +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; + +import java.util.Map; + +/** + * Wraps the JsonSchemaKafkaSerializer and JsonSchemaKafkaDeserializer. + */ +public class JsonSchemaSerde implements Serde { + + final private Serializer serializer; + final private Deserializer deserializer; + + protected JsonSchemaSerde(Serializer serializer, Deserializer deserializer) { + this.serializer = serializer; + this.deserializer = deserializer; + } + + @Override + public void configure(Map configs, boolean isKey) { + serializer.configure(configs, isKey); + deserializer.configure(configs, isKey); + } + + @Override + public void close() { + serializer.close(); + deserializer.close(); + } + + @Override + public Serializer serializer() { + return serializer; + } + + @Override + public Deserializer deserializer() { + return deserializer; + } +} diff --git a/serdes/kafka/protobuf-serde/.gitignore b/serdes/kafka/protobuf-serde/.gitignore new file mode 100644 index 0000000000..00d2ab71dd --- /dev/null +++ b/serdes/kafka/protobuf-serde/.gitignore @@ -0,0 +1,2 @@ +/.apt_generated/ +/.apt_generated_tests/ diff --git a/serdes/kafka/protobuf-serde/pom.xml b/serdes/kafka/protobuf-serde/pom.xml new file mode 100644 index 0000000000..71044f1d68 --- /dev/null +++ b/serdes/kafka/protobuf-serde/pom.xml @@ -0,0 +1,70 @@ + + + 4.0.0 + + io.apicurio + apicurio-registry + 3.0.0-SNAPSHOT + ../../../pom.xml + + + apicurio-registry-protobuf-serde-kafka + jar + apicurio-registry-protobuf-serde-kafka + + + ${project.basedir}/../../.. + + + + + io.apicurio + apicurio-registry-serde-kafka-common + + + + io.apicurio + apicurio-registry-serde-common-protobuf + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + + + detect + + initialize + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${proto-plugin.version} + true + + + gencode + + compile + + generate-sources + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + + + + + + + + + diff --git a/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializer.java b/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializer.java new file mode 100644 index 0000000000..a345ac3ad3 --- /dev/null +++ b/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializer.java @@ -0,0 +1,82 @@ +package io.apicurio.registry.serde.protobuf; + +import com.google.protobuf.Message; +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.AbstractDeserializer; +import io.apicurio.registry.serde.AbstractKafkaDeserializer; +import io.apicurio.registry.utils.protobuf.schema.ProtobufSchema; +import org.apache.kafka.common.header.Headers; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +public class ProtobufKafkaDeserializer + extends AbstractKafkaDeserializer { + + private ProtobufSerdeHeaders serdeHeaders; + + private ProtobufDeserializer protobufDeserializer; + + public ProtobufKafkaDeserializer() { + super(); + } + + public ProtobufKafkaDeserializer(RegistryClient client, + SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public ProtobufKafkaDeserializer(RegistryClient client) { + super(client); + } + + public ProtobufKafkaDeserializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + @Override + public void configure(Map configs, boolean isKey) { + serdeHeaders = new ProtobufSerdeHeaders(new HashMap<>(configs), isKey); + ProtobufDeserializerConfig protobufDeserializerConfig = new ProtobufDeserializerConfig(configs, + isKey); + this.protobufDeserializer = new ProtobufDeserializer<>(); + if (getSchemaResolver() != null) { + this.protobufDeserializer.setSchemaResolver(getSchemaResolver()); + } + protobufDeserializer.configure(protobufDeserializerConfig, isKey); + + super.configure(configs, isKey); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + */ + @Override + public SchemaParser schemaParser() { + return protobufDeserializer.schemaParser(); + } + + /** + * @see AbstractDeserializer#readData(io.apicurio.registry.resolver.ParsedSchema, java.nio.ByteBuffer, + * int, int) + */ + @Override + protected U readData(ParsedSchema schema, ByteBuffer buffer, int start, int length) { + return protobufDeserializer.readData(schema, buffer, start, length); + } + + @Override + public U deserialize(String topic, Headers headers, byte[] data) { + String messageTypeHeader = serdeHeaders.getMessageType(headers); + + if (messageTypeHeader != null) { + protobufDeserializer.setMessageTypeName(messageTypeHeader); + } + + return super.deserialize(topic, headers, data); + } +} diff --git a/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializer.java b/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializer.java new file mode 100644 index 0000000000..e60860be78 --- /dev/null +++ b/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializer.java @@ -0,0 +1,88 @@ +package io.apicurio.registry.serde.protobuf; + +import com.google.protobuf.Message; +import io.apicurio.registry.resolver.ParsedSchema; +import io.apicurio.registry.resolver.SchemaParser; +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.AbstractKafkaSerializer; +import io.apicurio.registry.utils.protobuf.schema.ProtobufSchema; +import org.apache.kafka.common.header.Headers; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; + +public class ProtobufKafkaSerializer extends AbstractKafkaSerializer { + + private ProtobufSerdeHeaders serdeHeaders; + private ProtobufSerializer protobufSerializer; + + public ProtobufKafkaSerializer() { + super(); + } + + public ProtobufKafkaSerializer(RegistryClient client, + ArtifactReferenceResolverStrategy artifactResolverStrategy, + SchemaResolver schemaResolver) { + super(client, artifactResolverStrategy, schemaResolver); + } + + public ProtobufKafkaSerializer(RegistryClient client) { + super(client); + } + + public ProtobufKafkaSerializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + @Override + public void configure(Map configs, boolean isKey) { + ProtobufSerializerConfig config = new ProtobufSerializerConfig(configs); + serdeHeaders = new ProtobufSerdeHeaders(new HashMap<>(configs), isKey); + this.protobufSerializer = new ProtobufSerializer<>(); + if (getSchemaResolver() != null) { + this.protobufSerializer.setSchemaResolver(getSchemaResolver()); + } + protobufSerializer.configure(config, isKey); + + super.configure(configs, isKey); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerDe#schemaParser() + */ + @Override + public SchemaParser schemaParser() { + return protobufSerializer.schemaParser(); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(io.apicurio.registry.resolver.ParsedSchema, + * java.lang.Object, java.io.OutputStream) + */ + @Override + protected void serializeData(ParsedSchema schema, U data, OutputStream out) + throws IOException { + protobufSerializer.serializeData(schema, data, out); + } + + /** + * @see io.apicurio.registry.serde.AbstractKafkaSerializer#serializeData(org.apache.kafka.common.header.Headers, + * io.apicurio.registry.resolver.ParsedSchema, java.lang.Object, java.io.OutputStream) + */ + @Override + protected void serializeData(Headers headers, ParsedSchema schema, U data, + OutputStream out) throws IOException { + if (headers != null) { + serdeHeaders.addMessageTypeHeader(headers, data.getClass().getName()); + serdeHeaders.addProtobufTypeNameHeader(headers, data.getDescriptorForType().getName()); + } else { + protobufSerializer.setWriteRef(false); + } + + serializeData(schema, data, out); + } +} diff --git a/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerde.java b/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerde.java new file mode 100644 index 0000000000..eaf7f29047 --- /dev/null +++ b/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerde.java @@ -0,0 +1,44 @@ +package io.apicurio.registry.serde.protobuf; + +import com.google.protobuf.Message; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serializer; + +import java.util.Map; + +/**** + * Wraps the ProtobufKafkaSerializer and ProtobufKafkaDeserializer. + */ +public class ProtobufSerde implements Serde { + + final private Serializer serializer; + final private Deserializer deserializer; + + protected ProtobufSerde(Serializer serializer, Deserializer deserializer) { + this.serializer = serializer; + this.deserializer = deserializer; + } + + @Override + public void configure(Map configs, boolean isKey) { + serializer.configure(configs, isKey); + deserializer.configure(configs, isKey); + } + + @Override + public void close() { + serializer.close(); + deserializer.close(); + } + + @Override + public Serializer serializer() { + return serializer; + } + + @Override + public Deserializer deserializer() { + return deserializer; + } +} diff --git a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerdeHeaders.java b/serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerdeHeaders.java similarity index 100% rename from serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerdeHeaders.java rename to serdes/kafka/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerdeHeaders.java diff --git a/serdes/kafka/protobuf-serde/src/main/proto/ref.proto b/serdes/kafka/protobuf-serde/src/main/proto/ref.proto new file mode 100644 index 0000000000..8204831050 --- /dev/null +++ b/serdes/kafka/protobuf-serde/src/main/proto/ref.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package io.apicurio.registry.serde.protobuf.ref; + +option optimize_for = SPEED; +option java_package = "io.apicurio.registry.serde.protobuf.ref"; + +message Ref { + string name = 1; +} \ No newline at end of file diff --git a/serdes/kafka/serde-kafka-common/pom.xml b/serdes/kafka/serde-kafka-common/pom.xml new file mode 100644 index 0000000000..5a1225e44b --- /dev/null +++ b/serdes/kafka/serde-kafka-common/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + io.apicurio + apicurio-registry + 3.0.0-SNAPSHOT + ../../../pom.xml + + + apicurio-registry-serde-kafka-common + jar + apicurio-registry-serde-kafka-common + + + ${project.basedir}/../../.. + + + + + + io.apicurio + apicurio-registry-serde-common + + + + org.slf4j + slf4j-api + + + + org.jboss.slf4j + slf4j-jboss-logging + ${jboss-slf4j.version} + + + + org.apache.kafka + kafka-clients + + + + org.junit.jupiter + junit-jupiter + test + + + + diff --git a/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaDeserializer.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaDeserializer.java new file mode 100644 index 0000000000..d3b0a23a4b --- /dev/null +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaDeserializer.java @@ -0,0 +1,92 @@ +package io.apicurio.registry.serde; + +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReference; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.resolver.utils.Utils; +import io.apicurio.registry.rest.client.RegistryClient; +import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; +import io.apicurio.registry.serde.headers.HeadersHandler; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.serialization.Deserializer; + +import java.util.Map; + +public abstract class AbstractKafkaDeserializer extends AbstractDeserializer + implements Deserializer { + + protected HeadersHandler headersHandler; + + public AbstractKafkaDeserializer() { + super(); + } + + public AbstractKafkaDeserializer(RegistryClient client) { + super(client); + } + + public AbstractKafkaDeserializer(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public AbstractKafkaDeserializer(RegistryClient client, SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public AbstractKafkaDeserializer(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + super(client, strategy, schemaResolver); + } + + @Override + public void configure(Map configs, boolean isKey) { + super.configure(new SerdeConfig(configs), isKey); + + this.configure(new BaseKafkaSerDeConfig(configs), isKey); + } + + protected void configure(BaseKafkaSerDeConfig config, boolean isKey) { + super.configure(config.originals(), isKey, schemaParser()); + + boolean headersEnabled = config.enableHeaders(); + if (headersEnabled) { + Object headersHandler = config.getHeadersHandler(); + Utils.instantiate(HeadersHandler.class, headersHandler, this::setHeadersHandler); + this.headersHandler.configure(config.originals(), isKey); + } + } + + public void setHeadersHandler(HeadersHandler headersHandler) { + this.headersHandler = headersHandler; + } + + @Override + public U deserialize(String topic, byte[] data) { + return deserializeData(topic, data); + } + + @Override + public U deserialize(String topic, Headers headers, byte[] data) { + if (data == null) { + return null; + } + ArtifactReference artifactReference = null; + if (headersHandler != null && headers != null) { + artifactReference = headersHandler.readHeaders(headers); + + if (artifactReference.hasValue()) { + return readData(topic, data, artifactReference); + } + } + if (data[0] == MAGIC_BYTE) { + return deserialize(topic, data); + } else if (headers == null) { + throw new IllegalStateException("Headers cannot be null"); + } else { + // try to read data even if artifactReference has no value, maybe there is a + // fallbackArtifactProvider configured + return readData(topic, data, artifactReference); + } + } +} diff --git a/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerDe.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerDe.java new file mode 100644 index 0000000000..c62dfe7a81 --- /dev/null +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerDe.java @@ -0,0 +1,32 @@ +package io.apicurio.registry.serde; + +import io.apicurio.registry.resolver.SchemaResolver; +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.rest.client.RegistryClient; + +/** + * Common class for both serializer and deserializer. + */ +public abstract class AbstractKafkaSerDe extends AbstractSerDe { + + public AbstractKafkaSerDe() { + super(); + } + + public AbstractKafkaSerDe(RegistryClient client) { + super(client); + } + + public AbstractKafkaSerDe(SchemaResolver schemaResolver) { + super(schemaResolver); + } + + public AbstractKafkaSerDe(RegistryClient client, SchemaResolver schemaResolver) { + super(client, schemaResolver); + } + + public AbstractKafkaSerDe(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + super(client, strategy, schemaResolver); + } +} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerializer.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerializer.java similarity index 55% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerializer.java rename to serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerializer.java index c7f9be3608..8f5eeb6b5a 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerializer.java +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerializer.java @@ -4,10 +4,13 @@ import io.apicurio.registry.resolver.SchemaLookupResult; import io.apicurio.registry.resolver.SchemaResolver; import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.apicurio.registry.resolver.utils.Utils; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; +import io.apicurio.registry.serde.config.SerdeConfig; import io.apicurio.registry.serde.data.KafkaSerdeMetadata; -import io.apicurio.registry.serde.data.KafkaSerdeRecord; +import io.apicurio.registry.serde.data.SerdeRecord; +import io.apicurio.registry.serde.headers.HeadersHandler; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.serialization.Serializer; @@ -17,9 +20,11 @@ import java.io.UncheckedIOException; import java.util.Map; -public abstract class AbstractKafkaSerializer extends AbstractKafkaSerDe +public abstract class AbstractKafkaSerializer extends AbstractSerializer implements Serializer { + protected HeadersHandler headersHandler; + public AbstractKafkaSerializer() { super(); } @@ -32,29 +37,39 @@ public AbstractKafkaSerializer(SchemaResolver schemaResolver) { super(schemaResolver); } - public AbstractKafkaSerializer(RegistryClient client, - ArtifactReferenceResolverStrategy artifactResolverStrategy, - SchemaResolver schemaResolver) { + public AbstractKafkaSerializer(RegistryClient client, SchemaResolver schemaResolver) { super(client, schemaResolver); - getSchemaResolver().setArtifactResolverStrategy(artifactResolverStrategy); + } + + public AbstractKafkaSerializer(RegistryClient client, ArtifactReferenceResolverStrategy strategy, + SchemaResolver schemaResolver) { + super(client, strategy, schemaResolver); } @Override public void configure(Map configs, boolean isKey) { - super.configure(new BaseKafkaSerDeConfig(configs), isKey); + super.configure(new SerdeConfig(configs), isKey); + this.configure(new BaseKafkaSerDeConfig(configs), isKey); } - protected abstract void serializeData(ParsedSchema schema, U data, OutputStream out) - throws IOException; + protected void configure(BaseKafkaSerDeConfig config, boolean isKey) { + super.configure(config.originals(), isKey, schemaParser()); - protected abstract void serializeData(Headers headers, ParsedSchema schema, U data, OutputStream out) - throws IOException; + boolean headersEnabled = config.enableHeaders(); + if (headersEnabled) { + Object headersHandler = config.getHeadersHandler(); + Utils.instantiate(HeadersHandler.class, headersHandler, this::setHeadersHandler); + this.headersHandler.configure(config.originals(), isKey); + } + } - @Override - public byte[] serialize(String topic, U data) { - return serialize(topic, null, data); + public void setHeadersHandler(HeadersHandler headersHandler) { + this.headersHandler = headersHandler; } + protected abstract void serializeData(Headers headers, ParsedSchema schema, U data, OutputStream out) + throws IOException; + @Override public byte[] serialize(String topic, Headers headers, U data) { // just return null @@ -62,33 +77,24 @@ public byte[] serialize(String topic, Headers headers, U data) { return null; } try { - - KafkaSerdeMetadata resolverMetadata = new KafkaSerdeMetadata(topic, isKey(), headers); - - SchemaLookupResult schema = getSchemaResolver() - .resolveSchema(new KafkaSerdeRecord<>(resolverMetadata, data)); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); if (headersHandler != null && headers != null) { + KafkaSerdeMetadata resolverMetadata = new KafkaSerdeMetadata(topic, isKey(), headers); + SchemaLookupResult schema = getSchemaResolver() + .resolveSchema(new SerdeRecord<>(resolverMetadata, data)); + ByteArrayOutputStream out = new ByteArrayOutputStream(); headersHandler.writeHeaders(headers, schema.toArtifactReference()); serializeData(headers, schema.getParsedSchema(), data, out); + return out.toByteArray(); } else { - out.write(MAGIC_BYTE); - getIdHandler().writeId(schema.toArtifactReference(), out); - serializeData(schema.getParsedSchema(), data, out); + return serializeData(topic, data); } - return out.toByteArray(); } catch (IOException e) { throw new UncheckedIOException(e); } } @Override - public void close() { - try { - this.schemaResolver.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + public byte[] serialize(String topic, U data) { + return serialize(topic, null, data); } } diff --git a/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaSerDeConfig.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaSerDeConfig.java new file mode 100644 index 0000000000..d5dc82ac10 --- /dev/null +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaSerDeConfig.java @@ -0,0 +1,51 @@ +package io.apicurio.registry.serde.config; + +import java.util.HashMap; +import java.util.Map; + +import static io.apicurio.registry.serde.config.KafkaSerdeConfig.ENABLE_HEADERS; +import static io.apicurio.registry.serde.config.KafkaSerdeConfig.ENABLE_HEADERS_DEFAULT; +import static io.apicurio.registry.serde.config.KafkaSerdeConfig.HEADERS_HANDLER; +import static io.apicurio.registry.serde.config.KafkaSerdeConfig.HEADERS_HANDLER_DEFAULT; +import static io.apicurio.registry.serde.config.SerdeConfig.*; +import static java.util.Map.entry; + +public class BaseKafkaSerDeConfig extends SerdeConfig { + + public BaseKafkaSerDeConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + } + + public BaseKafkaSerDeConfig() { + this.originals = getDefaults(); + } + + public Object getIdHandler() { + return this.getObject(ID_HANDLER); + } + + public boolean enableHeaders() { + return this.getBoolean(ENABLE_HEADERS); + } + + public Object getHeadersHandler() { + return this.getObject(HEADERS_HANDLER); + } + + public IdOption useIdOption() { + return IdOption.valueOf(this.getString(USE_ID)); + } + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } + + private static final Map DEFAULTS = Map.ofEntries( + entry(ENABLE_HEADERS, ENABLE_HEADERS_DEFAULT), entry(HEADERS_HANDLER, HEADERS_HANDLER_DEFAULT)); + +} diff --git a/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/config/KafkaSerdeConfig.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/config/KafkaSerdeConfig.java new file mode 100644 index 0000000000..1e7ccd5f72 --- /dev/null +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/config/KafkaSerdeConfig.java @@ -0,0 +1,109 @@ +package io.apicurio.registry.serde.config; + +import io.apicurio.registry.serde.headers.DefaultHeadersHandler; +import io.apicurio.registry.serde.headers.HeadersHandler; +import io.apicurio.registry.serde.headers.KafkaSerdeHeaders; + +public class KafkaSerdeConfig { + + /** + * Boolean to indicate whether serde classes should pass Global Id information via message headers instead + * of in the message payload. + */ + public static final String ENABLE_HEADERS = "apicurio.registry.headers.enabled"; + public static final boolean ENABLE_HEADERS_DEFAULT = false; + + /** + * Fully qualified Java classname of a class that implements {@link HeadersHandler} and is responsible for + * writing the schema's Global ID to the message headers. Only used when + * {@link KafkaSerdeConfig#ENABLE_HEADERS} is 'true'. + */ + public static final String HEADERS_HANDLER = "apicurio.registry.headers.handler"; + public static final String HEADERS_HANDLER_DEFAULT = DefaultHeadersHandler.class.getName(); + + /** + * Used to override the Kafka message header name used to pass the groupId for the message key. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_KEY_GROUP_ID}. + */ + public static final String HEADER_KEY_GROUP_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.groupId.name"; + /** + * Used to override the Kafka message header name used to pass the groupId for the message value. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_VALUE_GROUP_ID}. + */ + public static final String HEADER_VALUE_GROUP_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.groupId.name"; + /** + * Used to override the Kafka message header name used to pass the artifactId for the message key. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_KEY_ARTIFACT_ID}. + */ + public static final String HEADER_KEY_ARTIFACT_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.artifactId.name"; + /** + * Used to override the Kafka message header name used to pass the artifactId for the message value. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_VALUE_ARTIFACT_ID}. + */ + public static final String HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.artifactId.name"; + /** + * Used to override the Kafka message header name used to pass the version for the message key. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_KEY_VERSION}. + */ + public static final String HEADER_KEY_VERSION_OVERRIDE_NAME = "apicurio.registry.headers.key.version.name"; + /** + * Used to override the Kafka message header name used to pass the version for the message value. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_VALUE_VERSION}. + */ + public static final String HEADER_VALUE_VERSION_OVERRIDE_NAME = "apicurio.registry.headers.value.version.name"; + /** + * Used to override the Kafka message header name used to pass the globalId for the message key. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_KEY_GLOBAL_ID}. + */ + public static final String HEADER_KEY_GLOBAL_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.globalId.name"; + /** + * Used to override the Kafka message header name used to pass the globalId for the message value. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_VALUE_GLOBAL_ID}. + */ + public static final String HEADER_VALUE_GLOBAL_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.globalId.name"; + /** + * Used to override the Kafka message header name used to pass the contentId for the message key. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_KEY_CONTENT_ID}. + */ + public static final String HEADER_KEY_CONTENT_ID_OVERRIDE_NAME = "apicurio.registry.headers.key.contentId.name"; + /** + * Used to override the Kafka message header name used to pass the contentId for the message value. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_VALUE_CONTENT_ID}. + */ + public static final String HEADER_VALUE_CONTENT_ID_OVERRIDE_NAME = "apicurio.registry.headers.value.contentId.name"; + /** + * Used to override the Kafka message header name used to pass the contentHash for the message key. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_KEY_CONTENT_HASH}. + */ + public static final String HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME = "apicurio.registry.headers.key.contentHash.name"; + /** + * Used to override the Kafka message header name used to pass the contentHash for the message value. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Default value is + * {@link KafkaSerdeHeaders#HEADER_VALUE_CONTENT_HASH}. + */ + public static final String HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME = "apicurio.registry.headers.value.contentHash.name"; + /** + * Used to override the Kafka message header name used to pass the message type for the message key. Only + * applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Only used by the JSON Schema serde + * classes. Default value is {@link KafkaSerdeHeaders#HEADER_KEY_MESSAGE_TYPE}. + */ + public static final String HEADER_KEY_MESSAGE_TYPE_OVERRIDE_NAME = "apicurio.registry.headers.key.msgType.name"; + /** + * Used to override the Kafka message header name used to pass the message type for the message value. + * Only applicable when {@link KafkaSerdeConfig#ENABLE_HEADERS} is enabled. Only used by the JSON Schema + * serde classes. Default value is {@link KafkaSerdeHeaders#HEADER_VALUE_MESSAGE_TYPE}. + */ + public static final String HEADER_VALUE_MESSAGE_TYPE_OVERRIDE_NAME = "apicurio.registry.headers.value.msgType.name"; + +} diff --git a/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeMetadata.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeMetadata.java new file mode 100644 index 0000000000..7a2cf9ec9f --- /dev/null +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/data/KafkaSerdeMetadata.java @@ -0,0 +1,33 @@ +package io.apicurio.registry.serde.data; + +import io.apicurio.registry.resolver.strategy.ArtifactReference; +import org.apache.kafka.common.header.Headers; + +/** + * Kafka specific implementation for the Record Metadata abstraction used by the SchemaResolver + */ +public class KafkaSerdeMetadata extends SerdeMetadata { + + private final Headers headers; + + public KafkaSerdeMetadata(String topic, boolean isKey, Headers headers) { + super(topic, isKey); + this.headers = headers; + } + + /** + * @see io.apicurio.registry.resolver.data.Metadata#artifactReference() + */ + @Override + public ArtifactReference artifactReference() { + return null; + } + + /** + * @return the headers + */ + public Headers getHeaders() { + return headers; + } + +} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandler.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandler.java similarity index 97% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandler.java rename to serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandler.java index be946db34e..636d4078d2 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandler.java +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandler.java @@ -41,12 +41,14 @@ public void configure(Map configs, boolean isKey) { artifactIdHeaderName = config.getValueArtifactIdHeader(); versionHeaderName = config.getValueVersionHeader(); } - idOption = config.useIdOption(); + if (config.useIdOption() != null) { + idOption = config.useIdOption(); + } } /** * @see io.apicurio.registry.serde.headers.HeadersHandler#writeHeaders(org.apache.kafka.common.header.Headers, - * io.apicurio.registry.serde.SchemaLookupResult) + * io.apicurio.registry.resolver.SchemaLookupResult) */ @Override public void writeHeaders(Headers headers, ArtifactReference reference) { diff --git a/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerConfig.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerConfig.java new file mode 100644 index 0000000000..1026103420 --- /dev/null +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerConfig.java @@ -0,0 +1,90 @@ +package io.apicurio.registry.serde.headers; + +import io.apicurio.registry.serde.config.SerdeConfig; + +import java.util.HashMap; +import java.util.Map; + +import static io.apicurio.registry.serde.config.KafkaSerdeConfig.*; +import static io.apicurio.registry.serde.headers.KafkaSerdeHeaders.*; +import static java.util.Map.entry; + +public class DefaultHeadersHandlerConfig extends SerdeConfig { + + private final Map DEFAULTS = Map.ofEntries( + entry(HEADER_KEY_GLOBAL_ID_OVERRIDE_NAME, HEADER_KEY_GLOBAL_ID), + entry(HEADER_KEY_CONTENT_ID_OVERRIDE_NAME, HEADER_KEY_CONTENT_ID), + entry(HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, HEADER_KEY_CONTENT_HASH), + entry(HEADER_KEY_GROUP_ID_OVERRIDE_NAME, HEADER_KEY_GROUP_ID), + entry(HEADER_KEY_ARTIFACT_ID_OVERRIDE_NAME, HEADER_KEY_ARTIFACT_ID), + entry(HEADER_KEY_VERSION_OVERRIDE_NAME, HEADER_KEY_VERSION), + entry(HEADER_VALUE_GLOBAL_ID_OVERRIDE_NAME, HEADER_VALUE_GLOBAL_ID), + entry(HEADER_VALUE_CONTENT_ID_OVERRIDE_NAME, HEADER_VALUE_CONTENT_ID), + entry(HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, HEADER_VALUE_CONTENT_HASH), + entry(HEADER_VALUE_GROUP_ID_OVERRIDE_NAME, HEADER_VALUE_GROUP_ID), + entry(HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME, HEADER_VALUE_ARTIFACT_ID), + entry(HEADER_VALUE_VERSION_OVERRIDE_NAME, HEADER_VALUE_VERSION) + + ); + + public DefaultHeadersHandlerConfig(Map originals) { + Map joint = new HashMap<>(getDefaults()); + joint.putAll(originals); + this.originals = joint; + } + + public String getKeyGlobalIdHeader() { + return this.getString(HEADER_KEY_GLOBAL_ID_OVERRIDE_NAME); + } + + public String getKeyContentIdHeader() { + return this.getString(HEADER_KEY_CONTENT_ID_OVERRIDE_NAME); + } + + public String getKeyContentHashHeader() { + return this.getString(HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME); + } + + public String getKeyGroupIdHeader() { + return this.getString(HEADER_KEY_GROUP_ID_OVERRIDE_NAME); + } + + public String getKeyArtifactIdHeader() { + return this.getString(HEADER_KEY_ARTIFACT_ID_OVERRIDE_NAME); + } + + public String getKeyVersionHeader() { + return this.getString(HEADER_KEY_VERSION_OVERRIDE_NAME); + } + + public String getValueGlobalIdHeader() { + return this.getString(HEADER_VALUE_GLOBAL_ID_OVERRIDE_NAME); + } + + public String getValueContentIdHeader() { + return this.getString(HEADER_VALUE_CONTENT_ID_OVERRIDE_NAME); + } + + public String getValueContentHashHeader() { + return this.getString(HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME); + } + + public String getValueGroupIdHeader() { + return this.getString(HEADER_VALUE_GROUP_ID_OVERRIDE_NAME); + } + + public String getValueArtifactIdHeader() { + return this.getString(HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME); + } + + public String getValueVersionHeader() { + return this.getString(HEADER_VALUE_VERSION_OVERRIDE_NAME); + } + + @Override + protected Map getDefaults() { + Map joint = new HashMap<>(super.getDefaults()); + joint.putAll(DEFAULTS); + return joint; + } +} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/HeadersHandler.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/HeadersHandler.java similarity index 82% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/HeadersHandler.java rename to serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/HeadersHandler.java index b9fc88f18d..783e133dfd 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/HeadersHandler.java +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/HeadersHandler.java @@ -14,7 +14,7 @@ public interface HeadersHandler { default void configure(Map configs, boolean isKey) { } - public void writeHeaders(Headers headers, ArtifactReference reference); + void writeHeaders(Headers headers, ArtifactReference reference); /** * Reads the kafka message headers and returns an ArtifactReference that can contain or not information to @@ -23,6 +23,6 @@ default void configure(Map configs, boolean isKey) { * @param headers * @return ArtifactReference */ - public ArtifactReference readHeaders(Headers headers); + ArtifactReference readHeaders(Headers headers); } diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SerdeHeaders.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/KafkaSerdeHeaders.java similarity index 95% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/SerdeHeaders.java rename to serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/KafkaSerdeHeaders.java index 70bc9f1c5c..d116a55a7f 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SerdeHeaders.java +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/KafkaSerdeHeaders.java @@ -1,10 +1,10 @@ -package io.apicurio.registry.serde; +package io.apicurio.registry.serde.headers; /** * Contains all of the header names used when serde classes are configured to pass information via headers * instead of via the message payload. Note that these header names can be overridden via configuration. */ -public class SerdeHeaders { +public class KafkaSerdeHeaders { public static final String HEADER_KEY_ENCODING = "apicurio.key.encoding"; public static final String HEADER_VALUE_ENCODING = "apicurio.value.encoding"; diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/MessageTypeSerdeHeaders.java b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/MessageTypeSerdeHeaders.java similarity index 72% rename from serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/MessageTypeSerdeHeaders.java rename to serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/MessageTypeSerdeHeaders.java index 7554eb1eba..a05a445688 100644 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/MessageTypeSerdeHeaders.java +++ b/serdes/kafka/serde-kafka-common/src/main/java/io/apicurio/registry/serde/headers/MessageTypeSerdeHeaders.java @@ -1,7 +1,6 @@ package io.apicurio.registry.serde.headers; -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.SerdeHeaders; +import io.apicurio.registry.serde.config.KafkaSerdeConfig; import io.apicurio.registry.utils.IoUtil; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; @@ -10,7 +9,7 @@ /** * Common utility class for serializers and deserializers that use config properties such as - * {@link SerdeConfig#HEADER_VALUE_MESSAGE_TYPE_OVERRIDE_NAME} + * {@link KafkaSerdeConfig#HEADER_VALUE_MESSAGE_TYPE_OVERRIDE_NAME} */ public class MessageTypeSerdeHeaders { @@ -19,11 +18,12 @@ public class MessageTypeSerdeHeaders { public MessageTypeSerdeHeaders(Map configs, boolean isKey) { if (isKey) { messageTypeHeaderName = (String) configs.getOrDefault( - SerdeConfig.HEADER_KEY_MESSAGE_TYPE_OVERRIDE_NAME, SerdeHeaders.HEADER_KEY_MESSAGE_TYPE); + KafkaSerdeConfig.HEADER_KEY_MESSAGE_TYPE_OVERRIDE_NAME, + KafkaSerdeHeaders.HEADER_KEY_MESSAGE_TYPE); } else { messageTypeHeaderName = (String) configs.getOrDefault( - SerdeConfig.HEADER_VALUE_MESSAGE_TYPE_OVERRIDE_NAME, - SerdeHeaders.HEADER_VALUE_MESSAGE_TYPE); + KafkaSerdeConfig.HEADER_VALUE_MESSAGE_TYPE_OVERRIDE_NAME, + KafkaSerdeHeaders.HEADER_VALUE_MESSAGE_TYPE); } } diff --git a/serdes/serde-common/src/test/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerTest.java b/serdes/kafka/serde-kafka-common/src/test/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerTest.java similarity index 82% rename from serdes/serde-common/src/test/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerTest.java rename to serdes/kafka/serde-kafka-common/src/test/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerTest.java index 676aacde4e..44a1fd32f7 100644 --- a/serdes/serde-common/src/test/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerTest.java +++ b/serdes/kafka/serde-kafka-common/src/test/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerTest.java @@ -1,7 +1,7 @@ package io.apicurio.registry.serde.headers; import io.apicurio.registry.resolver.strategy.ArtifactReference; -import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.config.KafkaSerdeConfig; import io.apicurio.registry.utils.IoUtil; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.header.internals.RecordHeaders; @@ -21,7 +21,7 @@ void testReadKeyHeadersHandlesPresentContentHash() { String contentHashHeaderName = "some key header name"; String contentHashValue = "context hash value"; Map configs = Collections - .singletonMap(SerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + .singletonMap(KafkaSerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(new RecordHeader[] { new RecordHeader(contentHashHeaderName, IoUtil.toBytes(contentHashValue)) }); DefaultHeadersHandler handler = new DefaultHeadersHandler(); @@ -36,7 +36,7 @@ void testReadKeyHeadersHandlesPresentContentHash() { void testReadKeyHeadersHandlesMissingContentHash() { String contentHashHeaderName = "another key header name"; Map configs = Collections - .singletonMap(SerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + .singletonMap(KafkaSerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(new RecordHeader[] {}); DefaultHeadersHandler handler = new DefaultHeadersHandler(); handler.configure(configs, true); @@ -50,8 +50,8 @@ void testReadKeyHeadersHandlesMissingContentHash() { void testReadValueHeadersHandlesPresentContentHash() { String contentHashHeaderName = "value header name"; String contentHashValue = "some value"; - Map configs = Collections - .singletonMap(SerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + Map configs = Collections.singletonMap( + KafkaSerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(new RecordHeader[] { new RecordHeader(contentHashHeaderName, IoUtil.toBytes(contentHashValue)) }); DefaultHeadersHandler handler = new DefaultHeadersHandler(); @@ -65,8 +65,8 @@ void testReadValueHeadersHandlesPresentContentHash() { @Test void testReadValueHeadersHandlesMissingContentHash() { String contentHashHeaderName = "another value header name"; - Map configs = Collections - .singletonMap(SerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + Map configs = Collections.singletonMap( + KafkaSerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(new RecordHeader[] {}); DefaultHeadersHandler handler = new DefaultHeadersHandler(); handler.configure(configs, false); @@ -81,7 +81,7 @@ void testWriteKeyHeadersHandlesPresentContentHash() { String contentHashHeaderName = "write key header name"; String contentHashValue = "some write key value"; Map configs = Collections - .singletonMap(SerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + .singletonMap(KafkaSerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(); DefaultHeadersHandler handler = new DefaultHeadersHandler(); handler.configure(configs, true); @@ -96,7 +96,7 @@ void testWriteKeyHeadersHandlesPresentContentHash() { void testWriteKeyHeadersHandlesMissingContentHash() { String contentHashHeaderName = "another header name"; Map configs = Collections - .singletonMap(SerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + .singletonMap(KafkaSerdeConfig.HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(); DefaultHeadersHandler handler = new DefaultHeadersHandler(); handler.configure(configs, true); @@ -111,8 +111,8 @@ void testWriteKeyHeadersHandlesMissingContentHash() { void testWriteValueHeadersHandlesPresentContentHash() { String contentHashHeaderName = "write value header name"; String contentHashValue = "some write value"; - Map configs = Collections - .singletonMap(SerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + Map configs = Collections.singletonMap( + KafkaSerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(); DefaultHeadersHandler handler = new DefaultHeadersHandler(); handler.configure(configs, false); @@ -126,7 +126,7 @@ void testWriteValueHeadersHandlesPresentContentHash() { @Test void testWriteValueHeadersHandlesMissingContentHash() { String contentHashHeaderName = "another write key header name"; - Map configs = Map.of(SerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, + Map configs = Map.of(KafkaSerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); RecordHeaders headers = new RecordHeaders(); DefaultHeadersHandler handler = new DefaultHeadersHandler(); @@ -146,8 +146,8 @@ void testWritesArtifactCoordinatesWhenContentHashPresent() { String artifactIdValue = "a artifact ID"; Map configs = new HashMap<>(); - configs.put(SerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); - configs.put(SerdeConfig.HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME, artifactIdHeaderName); + configs.put(KafkaSerdeConfig.HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, contentHashHeaderName); + configs.put(KafkaSerdeConfig.HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME, artifactIdHeaderName); RecordHeaders headers = new RecordHeaders(); DefaultHeadersHandler handler = new DefaultHeadersHandler(); handler.configure(configs, false); diff --git a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializerConfig.java b/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializerConfig.java deleted file mode 100644 index 26b774cf58..0000000000 --- a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaDeserializerConfig.java +++ /dev/null @@ -1,55 +0,0 @@ -package io.apicurio.registry.serde.protobuf; - -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; - -import java.util.Map; - -import static io.apicurio.registry.serde.SerdeConfig.*; - -public class ProtobufKafkaDeserializerConfig extends BaseKafkaSerDeConfig { - - public static final String SPECIFIC_RETURN_CLASS_DOC = "A class generated by Protocol buffers that the message value should be deserialized to"; - - public static final String DERIVE_CLASS_FROM_SCHEMA = "apicurio.protobuf.derive.class"; - public static final String DERIVE_CLASS_FROM_SCHEMA_DOC = "Whether to derive the class based on `java_outer_classname` and `java_multiple_files` from the Protobuf schema."; - - private static ConfigDef configDef() { - ConfigDef configDef = new ConfigDef() - .define(DESERIALIZER_SPECIFIC_KEY_RETURN_CLASS, Type.CLASS, null, Importance.MEDIUM, - SPECIFIC_RETURN_CLASS_DOC) - .define(DESERIALIZER_SPECIFIC_VALUE_RETURN_CLASS, Type.CLASS, null, Importance.MEDIUM, - SPECIFIC_RETURN_CLASS_DOC) - .define(DERIVE_CLASS_FROM_SCHEMA, Type.BOOLEAN, false, Importance.MEDIUM, - DERIVE_CLASS_FROM_SCHEMA_DOC); - return configDef; - } - - private boolean isKey; - - /** - * Constructor. - * - * @param originals - */ - public ProtobufKafkaDeserializerConfig(Map originals, boolean isKey) { - super(configDef(), originals); - this.isKey = isKey; - - } - - public Class getSpecificReturnClass() { - if (isKey) { - return this.getClass(DESERIALIZER_SPECIFIC_KEY_RETURN_CLASS); - } else { - return this.getClass(DESERIALIZER_SPECIFIC_VALUE_RETURN_CLASS); - } - } - - public boolean deriveClass() { - return this.getBoolean(DERIVE_CLASS_FROM_SCHEMA); - } - -} diff --git a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializerConfig.java b/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializerConfig.java deleted file mode 100644 index dd023342e2..0000000000 --- a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufKafkaSerializerConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -package io.apicurio.registry.serde.protobuf; - -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; - -import java.util.Map; - -import static io.apicurio.registry.serde.SerdeConfig.VALIDATION_ENABLED; -import static io.apicurio.registry.serde.SerdeConfig.VALIDATION_ENABLED_DEFAULT; - -public class ProtobufKafkaSerializerConfig extends BaseKafkaSerDeConfig { - - private static ConfigDef configDef() { - ConfigDef configDef = new ConfigDef().define(VALIDATION_ENABLED, Type.BOOLEAN, - VALIDATION_ENABLED_DEFAULT, Importance.MEDIUM, - "Whether to validate the data being sent adheres to the schema being used"); - return configDef; - } - - /** - * Constructor. - * - * @param originals - */ - public ProtobufKafkaSerializerConfig(Map originals) { - super(configDef(), originals); - - } - - public boolean validationEnabled() { - return this.getBoolean(VALIDATION_ENABLED); - } - -} diff --git a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerde.java b/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerde.java deleted file mode 100644 index 9a39d332f8..0000000000 --- a/serdes/protobuf-serde/src/main/java/io/apicurio/registry/serde/protobuf/ProtobufSerde.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.apicurio.registry.serde.protobuf; - -import com.google.protobuf.Message; -import io.apicurio.registry.serde.AbstractSerde; - -/**** - * Wraps the ProtobufKafkaSerializer and ProtobufKafkaDeserializer. - */ -public class ProtobufSerde extends AbstractSerde { - @SuppressWarnings({ "unchecked", "rawtypes" }) - public ProtobufSerde() { - super(new ProtobufKafkaSerializer(), new ProtobufKafkaDeserializer()); - } -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaDeserializer.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaDeserializer.java deleted file mode 100644 index e881ccad3f..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaDeserializer.java +++ /dev/null @@ -1,155 +0,0 @@ -package io.apicurio.registry.serde; - -import io.apicurio.registry.resolver.ParsedSchema; -import io.apicurio.registry.resolver.SchemaLookupResult; -import io.apicurio.registry.resolver.SchemaResolver; -import io.apicurio.registry.resolver.strategy.ArtifactReference; -import io.apicurio.registry.resolver.utils.Utils; -import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.config.BaseKafkaDeserializerConfig; -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import io.apicurio.registry.serde.fallback.DefaultFallbackArtifactProvider; -import io.apicurio.registry.serde.fallback.FallbackArtifactProvider; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.common.serialization.Deserializer; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; - -public abstract class AbstractKafkaDeserializer extends AbstractKafkaSerDe - implements Deserializer { - - protected FallbackArtifactProvider fallbackArtifactProvider; - - public AbstractKafkaDeserializer() { - super(); - } - - public AbstractKafkaDeserializer(RegistryClient client) { - super(client); - } - - public AbstractKafkaDeserializer(SchemaResolver schemaResolver) { - super(schemaResolver); - } - - public AbstractKafkaDeserializer(RegistryClient client, SchemaResolver schemaResolver) { - super(client, schemaResolver); - } - - /** - * @see io.apicurio.registry.serde.AbstractKafkaSerDe#configure(io.apicurio.registry.serde.config.BaseKafkaSerDeConfig, - * boolean) - */ - @Override - protected void configure(BaseKafkaSerDeConfig config, boolean isKey) { - super.configure(config, isKey); - - BaseKafkaDeserializerConfig deserializerConfig = new BaseKafkaDeserializerConfig(config.originals()); - - Object fallbackProvider = deserializerConfig.getFallbackArtifactProvider(); - Utils.instantiate(FallbackArtifactProvider.class, fallbackProvider, - this::setFallbackArtifactProvider); - fallbackArtifactProvider.configure(config.originals(), isKey); - - if (fallbackArtifactProvider instanceof DefaultFallbackArtifactProvider) { - if (!((DefaultFallbackArtifactProvider) fallbackArtifactProvider).isConfigured()) { - // it's not configured, just remove it so it's not executed - fallbackArtifactProvider = null; - } - } - - } - - /** - * @param fallbackArtifactProvider the fallbackArtifactProvider to set - */ - public void setFallbackArtifactProvider(FallbackArtifactProvider fallbackArtifactProvider) { - this.fallbackArtifactProvider = fallbackArtifactProvider; - } - - protected abstract U readData(ParsedSchema schema, ByteBuffer buffer, int start, int length); - - protected abstract U readData(Headers headers, ParsedSchema schema, ByteBuffer buffer, int start, - int length); - - @Override - public U deserialize(String topic, byte[] data) { - if (data == null) { - return null; - } - - ByteBuffer buffer = getByteBuffer(data); - ArtifactReference artifactReference = getIdHandler().readId(buffer); - - SchemaLookupResult schema = resolve(topic, null, data, artifactReference); - - int length = buffer.limit() - 1 - getIdHandler().idSize(); - int start = buffer.position() + buffer.arrayOffset(); - - return readData(schema.getParsedSchema(), buffer, start, length); - } - - @Override - public U deserialize(String topic, Headers headers, byte[] data) { - if (data == null) { - return null; - } - ArtifactReference artifactReference = null; - if (headersHandler != null && headers != null) { - artifactReference = headersHandler.readHeaders(headers); - - if (artifactReference.hasValue()) { - return readData(topic, headers, data, artifactReference); - } - } - if (data[0] == MAGIC_BYTE) { - return deserialize(topic, data); - } else if (headers == null) { - throw new IllegalStateException("Headers cannot be null"); - } else { - // try to read data even if artifactReference has no value, maybe there is a - // fallbackArtifactProvider configured - return readData(topic, headers, data, artifactReference); - } - } - - private U readData(String topic, Headers headers, byte[] data, ArtifactReference artifactReference) { - SchemaLookupResult schema = resolve(topic, headers, data, artifactReference); - - ByteBuffer buffer = ByteBuffer.wrap(data); - int length = buffer.limit(); - int start = buffer.position(); - - return readData(headers, schema.getParsedSchema(), buffer, start, length); - } - - private SchemaLookupResult resolve(String topic, Headers headers, byte[] data, - ArtifactReference artifactReference) { - try { - return getSchemaResolver().resolveSchemaByArtifactReference(artifactReference); - } catch (RuntimeException e) { - if (fallbackArtifactProvider == null) { - throw e; - } else { - try { - ArtifactReference fallbackReference = fallbackArtifactProvider.get(topic, headers, data); - return getSchemaResolver().resolveSchemaByArtifactReference(fallbackReference); - } catch (RuntimeException fe) { - fe.addSuppressed(e); - throw fe; - } - } - } - } - - @Override - public void close() { - try { - this.schemaResolver.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerDe.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerDe.java deleted file mode 100644 index 4fa7dae9e8..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractKafkaSerDe.java +++ /dev/null @@ -1,99 +0,0 @@ -package io.apicurio.registry.serde; - -import io.apicurio.registry.resolver.SchemaParser; -import io.apicurio.registry.resolver.SchemaResolver; -import io.apicurio.registry.resolver.utils.Utils; -import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import io.apicurio.registry.serde.headers.HeadersHandler; -import org.apache.kafka.common.errors.SerializationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.Objects; - -/** - * Common class for both serializer and deserializer. - */ -public abstract class AbstractKafkaSerDe extends SchemaResolverConfigurer { - - protected final Logger log = LoggerFactory.getLogger(getClass()); - - public static final byte MAGIC_BYTE = 0x0; - protected boolean key; // do we handle key or value with this ser/de? - - protected IdHandler idHandler; - protected HeadersHandler headersHandler; - - public AbstractKafkaSerDe() { - super(); - } - - public AbstractKafkaSerDe(RegistryClient client) { - super(client); - } - - public AbstractKafkaSerDe(SchemaResolver schemaResolver) { - super(schemaResolver); - } - - public AbstractKafkaSerDe(RegistryClient client, SchemaResolver schemaResolver) { - super(client, schemaResolver); - } - - public abstract void configure(Map configs, boolean isKey); - - protected void configure(BaseKafkaSerDeConfig config, boolean isKey) { - super.configure(config.originals(), isKey, schemaParser()); - key = isKey; - if (idHandler == null) { - Object idh = config.getIdHandler(); - Utils.instantiate(IdHandler.class, idh, this::setIdHandler); - } - idHandler.configure(config.originals(), isKey); - - boolean headersEnabled = config.enableHeaders(); - if (headersEnabled) { - Object headersHandler = config.getHeadersHandler(); - Utils.instantiate(HeadersHandler.class, headersHandler, this::setHeadersHandler); - this.headersHandler.configure(config.originals(), isKey); - } - } - - public abstract SchemaParser schemaParser(); - - public IdHandler getIdHandler() { - return idHandler; - } - - public void setHeadersHandler(HeadersHandler headersHandler) { - this.headersHandler = headersHandler; - } - - public void setIdHandler(IdHandler idHandler) { - this.idHandler = Objects.requireNonNull(idHandler); - } - - public void as4ByteId() { - setIdHandler(new Default4ByteIdHandler()); - } - - public void reset() { - schemaResolver.reset(); - } - - protected boolean isKey() { - return key; - } - - public static ByteBuffer getByteBuffer(byte[] payload) { - ByteBuffer buffer = ByteBuffer.wrap(payload); - if (buffer.get() != MAGIC_BYTE) { - throw new SerializationException("Unknown magic byte!"); - } - return buffer; - } - -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSchemaResolver.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSchemaResolver.java deleted file mode 100644 index 73d1d883de..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/AbstractSchemaResolver.java +++ /dev/null @@ -1,323 +0,0 @@ -package io.apicurio.registry.serde; - -import com.microsoft.kiota.RequestAdapter; -import io.apicurio.registry.client.auth.VertXAuthFactory; -import io.apicurio.registry.resolver.ERCache; -import io.apicurio.registry.resolver.ParsedSchemaImpl; -import io.apicurio.registry.resolver.config.DefaultSchemaResolverConfig; -import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; -import io.apicurio.registry.resolver.utils.Utils; -import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.rest.client.models.VersionMetaData; -import io.apicurio.registry.serde.data.KafkaSerdeMetadata; -import io.apicurio.registry.serde.data.KafkaSerdeRecord; -import io.apicurio.registry.serde.strategy.ArtifactReference; -import io.apicurio.registry.utils.IoUtil; -import io.kiota.http.vertx.VertXRequestAdapter; -import io.vertx.core.Vertx; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.Map; -import java.util.Optional; - -import static io.apicurio.registry.client.auth.VertXAuthFactory.buildOIDCWebClient; -import static io.apicurio.registry.client.auth.VertXAuthFactory.buildSimpleAuthWebClient; - -/** - * This class is deprecated, it's recommended to migrate to the new implementation at - * {@link io.apicurio.registry.resolver.AbstractSchemaResolver} Base implementation of {@link SchemaResolver} - */ -@Deprecated -public abstract class AbstractSchemaResolver implements SchemaResolver { - - protected final ERCache> schemaCache = new ERCache<>(); - - protected io.apicurio.registry.resolver.SchemaParser schemaParser; - protected RegistryClient client; - protected boolean isKey; - protected ArtifactReferenceResolverStrategy artifactResolverStrategy; - - protected String explicitArtifactGroupId; - protected String explicitArtifactId; - protected String explicitArtifactVersion; - - protected Vertx vertx; - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Override - public void configure(Map configs, - io.apicurio.registry.resolver.SchemaParser schemaMapper) { - this.schemaParser = schemaMapper; - - if (this.vertx == null) { - this.vertx = VertXAuthFactory.defaultVertx; - } - - Object isKeyFromConfig = configs.get(SerdeConfig.IS_KEY); - // is key have to come always, we set it - configure(configs, (Boolean) isKeyFromConfig, new SchemaParser() { - - /** - * @see io.apicurio.registry.serde.SchemaParser#artifactType() - */ - @Override - public String artifactType() { - return schemaMapper.artifactType(); - } - - /** - * @see io.apicurio.registry.serde.SchemaParser#parseSchema(byte[]) - */ - @Override - public Object parseSchema(byte[] rawSchema) { - // Empty map passed as references. References are not supported when using this class. - return schemaMapper.parseSchema(rawSchema, Collections.emptyMap()); - } - - }); - } - - /** - * @see io.apicurio.registry.serde.SchemaResolver#configure(java.util.Map, boolean, - * io.apicurio.registry.serde.SchemaParser) - */ - @Override - public void configure(Map configs, boolean isKey, SchemaParser schemaParser) { - this.isKey = isKey; - - if (this.vertx == null) { - this.vertx = VertXAuthFactory.defaultVertx; - } - - DefaultSchemaResolverConfig config = new DefaultSchemaResolverConfig(configs); - if (client == null) { - String baseUrl = config.getRegistryUrl(); - if (baseUrl == null) { - throw new IllegalArgumentException( - "Missing registry base url, set " + SerdeConfig.REGISTRY_URL); - } - - String authServerURL = config.getAuthServiceUrl(); - String tokenEndpoint = config.getTokenEndpoint(); - - try { - if (authServerURL != null || tokenEndpoint != null) { - client = configureClientWithBearerAuthentication(config, baseUrl, authServerURL, - tokenEndpoint); - } else { - String username = config.getAuthUsername(); - - if (username != null) { - client = configureClientWithBasicAuth(config, baseUrl, username); - } else { - RequestAdapter adapter = new VertXRequestAdapter(this.vertx); - adapter.setBaseUrl(baseUrl); - client = new RegistryClient(adapter); - } - } - } catch (Exception e) { - throw new IllegalStateException(e); - } - } - - Object ais = config.getArtifactResolverStrategy(); - Utils.instantiate(ArtifactReferenceResolverStrategy.class, ais, this::setArtifactResolverStrategy); - - schemaCache.configureLifetime(config.getCheckPeriod()); - schemaCache.configureRetryBackoff(config.getRetryBackoff()); - schemaCache.configureRetryCount(config.getRetryCount()); - - schemaCache.configureGlobalIdKeyExtractor(SchemaLookupResult::getGlobalId); - schemaCache.configureContentKeyExtractor( - schema -> Optional.ofNullable(schema.getRawSchema()).map(IoUtil::toString).orElse(null)); - schemaCache.configureContentIdKeyExtractor(SchemaLookupResult::getContentId); - schemaCache.configureContentHashKeyExtractor(SchemaLookupResult::getContentHash); - schemaCache.configureArtifactCoordinatesKeyExtractor(SchemaLookupResult::toArtifactCoordinates); - - schemaCache.checkInitialized(); - - String groupIdOverride = config.getExplicitArtifactGroupId(); - if (groupIdOverride != null) { - this.explicitArtifactGroupId = groupIdOverride; - } - String artifactIdOverride = config.getExplicitArtifactId(); - if (artifactIdOverride != null) { - this.explicitArtifactId = artifactIdOverride; - } - String artifactVersionOverride = config.getExplicitArtifactVersion(); - if (artifactVersionOverride != null) { - this.explicitArtifactVersion = artifactVersionOverride; - } - } - - /** - * @param client the client to set - */ - @Override - public void setClient(RegistryClient client) { - this.client = client; - } - - /** - * @param artifactResolverStrategy the artifactResolverStrategy to set - */ - @Override - public void setArtifactResolverStrategy( - ArtifactReferenceResolverStrategy artifactResolverStrategy) { - this.artifactResolverStrategy = artifactResolverStrategy; - } - - /** - * @param isKey the isKey to set - */ - public void setIsKey(boolean isKey) { - this.isKey = isKey; - } - - /** - * @see io.apicurio.registry.resolver.SchemaResolver#getSchemaParser() - */ - @Override - public io.apicurio.registry.resolver.SchemaParser getSchemaParser() { - return this.schemaParser; - } - - /** - * Resolve an artifact reference given the topic name, message headers, data, and optional parsed schema. - * This will use the artifact resolver strategy and then override the values from that strategy with any - * explicitly configured values (groupId, artifactId, version). - * - * @param topic - * @param headers - * @param data - * @param parsedSchema - */ - protected ArtifactReference resolveArtifactReference(String topic, T data, ParsedSchema parsedSchema) { - - KafkaSerdeMetadata metadata = new KafkaSerdeMetadata(topic, isKey, null); - KafkaSerdeRecord record = new KafkaSerdeRecord(metadata, data); - - io.apicurio.registry.resolver.ParsedSchema ps = new ParsedSchemaImpl() - .setParsedSchema(parsedSchema.getParsedSchema()).setRawSchema(parsedSchema.getRawSchema()); - - io.apicurio.registry.resolver.strategy.ArtifactReference artifactReference = artifactResolverStrategy - .artifactReference(record, ps); - - return ArtifactReference.builder() - .groupId(this.explicitArtifactGroupId == null ? artifactReference.getGroupId() - : this.explicitArtifactGroupId) - .artifactId(this.explicitArtifactId == null ? artifactReference.getArtifactId() - : this.explicitArtifactId) - .version(this.explicitArtifactVersion == null ? artifactReference.getVersion() - : this.explicitArtifactVersion) - .build(); - } - - protected SchemaLookupResult resolveSchemaByGlobalId(long globalId) { - return schemaCache.getByGlobalId(globalId, globalIdKey -> { - // TODO getContentByGlobalId have to return some minumum metadata (groupId, artifactId and - // version) - // TODO or at least add some method to the api to return the version metadata by globalId - // ArtifactMetaData artifactMetadata = client.getArtifactMetaData("TODO", artifactId); - InputStream rawSchema = client.ids().globalIds().byGlobalId(globalIdKey).get(); - - byte[] schema = IoUtil.toBytes(rawSchema); - S parsed = schemaParser.parseSchema(schema, Collections.emptyMap()); - - SchemaLookupResult.SchemaLookupResultBuilder result = SchemaLookupResult.builder(); - - return result - // FIXME it's impossible to retrieve this info with only the globalId - // .groupId(null) - // .artifactId(null) - // .version(0) - .globalId(globalIdKey).rawSchema(schema).schema(parsed).build(); - }); - } - - /** - * @see io.apicurio.registry.serde.SchemaResolver#reset() - */ - @Override - public void reset() { - this.schemaCache.clear(); - } - - @Override - public void close() throws IOException { - if (this.vertx != null) { - this.vertx.close(); - } - } - - private RegistryClient configureClientWithBearerAuthentication(DefaultSchemaResolverConfig config, - String registryUrl, String authServerUrl, String tokenEndpoint) { - RequestAdapter auth; - if (authServerUrl != null) { - auth = configureAuthWithRealm(config, authServerUrl); - } else { - auth = configureAuthWithUrl(config, tokenEndpoint); - } - auth.setBaseUrl(registryUrl); - return new RegistryClient(auth); - } - - private RequestAdapter configureAuthWithRealm(DefaultSchemaResolverConfig config, String authServerUrl) { - final String realm = config.getAuthRealm(); - - if (realm == null) { - throw new IllegalArgumentException("Missing registry auth realm, set " + SerdeConfig.AUTH_REALM); - } - - final String tokenEndpoint = authServerUrl - + String.format(SerdeConfig.AUTH_SERVICE_URL_TOKEN_ENDPOINT, realm); - - return configureAuthWithUrl(config, tokenEndpoint); - } - - private RequestAdapter configureAuthWithUrl(DefaultSchemaResolverConfig config, String tokenEndpoint) { - final String clientId = config.getAuthClientId(); - - if (clientId == null) { - throw new IllegalArgumentException( - "Missing registry auth clientId, set " + SerdeConfig.AUTH_CLIENT_ID); - } - final String clientSecret = config.getAuthClientSecret(); - - if (clientSecret == null) { - throw new IllegalArgumentException( - "Missing registry auth secret, set " + SerdeConfig.AUTH_CLIENT_SECRET); - } - - RequestAdapter adapter = new VertXRequestAdapter( - buildOIDCWebClient(this.vertx, tokenEndpoint, clientId, clientSecret)); - return adapter; - } - - private RegistryClient configureClientWithBasicAuth(DefaultSchemaResolverConfig config, - String registryUrl, String username) { - - final String password = config.getAuthPassword(); - - if (password == null) { - throw new IllegalArgumentException( - "Missing registry auth password, set " + SerdeConfig.AUTH_PASSWORD); - } - - var adapter = new VertXRequestAdapter(buildSimpleAuthWebClient(this.vertx, username, password)); - - adapter.setBaseUrl(registryUrl); - return new RegistryClient(adapter); - } - - protected void loadFromArtifactMetaData(VersionMetaData artifactMetadata, - SchemaLookupResult.SchemaLookupResultBuilder resultBuilder) { - resultBuilder.globalId(artifactMetadata.getGlobalId()); - resultBuilder.contentId(artifactMetadata.getContentId()); - resultBuilder.groupId(artifactMetadata.getGroupId()); - resultBuilder.artifactId(artifactMetadata.getArtifactId()); - resultBuilder.version(String.valueOf(artifactMetadata.getVersion())); - } -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/ParsedSchema.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/ParsedSchema.java deleted file mode 100644 index aeb5ece354..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/ParsedSchema.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.apicurio.registry.serde; - -/** - * This interface is deprecated and eventually will be replaced by - * {@link io.apicurio.registry.resolver.ParsedSchema} - */ -@Deprecated -public interface ParsedSchema { - - /** - * @return the parsedSchema - */ - public T getParsedSchema(); - - /** - * @return the rawSchema - */ - public byte[] getRawSchema(); - -} \ No newline at end of file diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/ParsedSchemaImpl.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/ParsedSchemaImpl.java deleted file mode 100644 index 8cfd9f0239..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/ParsedSchemaImpl.java +++ /dev/null @@ -1,48 +0,0 @@ -package io.apicurio.registry.serde; - -/** - * This class is deprecated and eventually will be replaced by - * {@link io.apicurio.registry.resolver.ParsedSchemaImpl} - */ -@Deprecated -public class ParsedSchemaImpl implements ParsedSchema { - - private T parsedSchema; - private byte[] rawSchema; - - public ParsedSchemaImpl() { - // empty - } - - /** - * @return the parsedSchema - */ - @Override - public T getParsedSchema() { - return parsedSchema; - } - - /** - * @param parsedSchema the parsedSchema to set - */ - public ParsedSchemaImpl setParsedSchema(T parsedSchema) { - this.parsedSchema = parsedSchema; - return this; - } - - /** - * @return the rawSchema - */ - @Override - public byte[] getRawSchema() { - return rawSchema; - } - - /** - * @param rawSchema the rawSchema to set - */ - public ParsedSchemaImpl setRawSchema(byte[] rawSchema) { - this.rawSchema = rawSchema; - return this; - } -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaLookupResult.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaLookupResult.java deleted file mode 100644 index 0de4044183..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaLookupResult.java +++ /dev/null @@ -1,161 +0,0 @@ -package io.apicurio.registry.serde; - -import io.apicurio.registry.resolver.ParsedSchemaImpl; -import io.apicurio.registry.resolver.strategy.ArtifactCoordinates; -import io.apicurio.registry.serde.strategy.ArtifactReference; - -/** - * This class is deprecated and eventually will be replaced by - * {@link io.apicurio.registry.resolver.SchemaLookupResult} - */ -@Deprecated -public class SchemaLookupResult { - - private byte[] rawSchema; - private T schema; - - private long globalId; - private long contentId; - private String contentHash; - private String groupId; - private String artifactId; - private String version; - - private SchemaLookupResult() { - // empty initialize manually - } - - /** - * @return the rawSchema - */ - public byte[] getRawSchema() { - return rawSchema; - } - - /** - * @return the schema - */ - public T getSchema() { - return schema; - } - - /** - * @return the globalId - */ - public long getGlobalId() { - return globalId; - } - - /** - * @return the contentId - */ - public long getContentId() { - return contentId; - } - - /** - * @return the contentHash - */ - public String getContentHash() { - return contentHash; - } - - /** - * @return the groupId - */ - public String getGroupId() { - return groupId; - } - - /** - * @return the artifactId - */ - public String getArtifactId() { - return artifactId; - } - - /** - * @return the version - */ - public String getVersion() { - return version; - } - - public ArtifactReference toArtifactReference() { - return ArtifactReference.builder().globalId(this.getGlobalId()).contentId(this.getContentId()) - .contentHash(this.getContentHash()).groupId(this.getGroupId()) - .artifactId(this.getArtifactId()).version(this.getVersion()).build(); - } - - public ArtifactCoordinates toArtifactCoordinates() { - return ArtifactCoordinates.builder().groupId(this.getGroupId()).artifactId(this.getArtifactId()) - .version(this.getVersion()).build(); - } - - @SuppressWarnings("rawtypes") - public io.apicurio.registry.resolver.SchemaLookupResult toCompat() { - return io.apicurio.registry.resolver.SchemaLookupResult.builder().contentId(contentId) - .contentHash(contentHash).globalId(globalId).groupId(groupId).artifactId(artifactId) - .version(version) - .parsedSchema(new ParsedSchemaImpl<>().setParsedSchema(schema).setRawSchema(rawSchema)) - .build(); - } - - public static SchemaLookupResultBuilder builder() { - return new SchemaLookupResultBuilder(); - } - - public static class SchemaLookupResultBuilder { - - private SchemaLookupResult result; - - SchemaLookupResultBuilder() { - this.result = new SchemaLookupResult<>(); - } - - public SchemaLookupResultBuilder rawSchema(byte[] rawSchema) { - this.result.rawSchema = rawSchema; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResultBuilder schema(T schema) { - this.result.schema = schema; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResultBuilder globalId(long globalId) { - this.result.globalId = globalId; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResultBuilder contentId(long contentId) { - this.result.contentId = contentId; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResultBuilder contentHash(String contentHash) { - this.result.contentHash = contentHash; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResultBuilder groupId(String groupId) { - this.result.groupId = groupId; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResultBuilder artifactId(String artifactId) { - this.result.artifactId = artifactId; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResultBuilder version(String version) { - this.result.version = version; - return SchemaLookupResultBuilder.this; - } - - public SchemaLookupResult build() { - return this.result; - } - - } -} \ No newline at end of file diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaParser.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaParser.java deleted file mode 100644 index b7b615ea19..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaParser.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.apicurio.registry.serde; - -/** - * This class is deprecated and eventually will be replaced by - * {@link io.apicurio.registry.resolver.SchemaParser} - */ -@Deprecated -public interface SchemaParser { - - public String artifactType(); - - public S parseSchema(byte[] rawSchema); - -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolver.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolver.java deleted file mode 100644 index 03b277f5aa..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/SchemaResolver.java +++ /dev/null @@ -1,138 +0,0 @@ -package io.apicurio.registry.serde; - -import io.apicurio.registry.resolver.data.Record; -import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; -import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.serde.data.KafkaSerdeMetadata; -import io.apicurio.registry.serde.data.KafkaSerdeRecord; -import io.apicurio.registry.serde.strategy.ArtifactReference; -import io.apicurio.registry.serde.strategy.ArtifactResolverStrategy; -import org.apache.kafka.common.header.Headers; - -import java.io.Closeable; -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -/** - * This interface is kept for compatibility, It's recommended to migrate custom implementations to adhere the - * new interface {@link io.apicurio.registry.resolver.SchemaResolver} - */ -@SuppressWarnings({ "rawtypes", "unchecked" }) -@Deprecated -public interface SchemaResolver - extends io.apicurio.registry.resolver.SchemaResolver, Closeable { - - /** - * Configure, if supported. - * - * @param configs the configs - * @param isKey are we handling key or value - */ - @Deprecated - default void configure(Map configs, boolean isKey, SchemaParser schemaMapper) { - } - - @Override - @Deprecated - public void setClient(RegistryClient client); - - @Deprecated - default void setArtifactResolverStrategy(ArtifactResolverStrategy artifactResolverStrategy) { - setArtifactResolverStrategy((ArtifactReferenceResolverStrategy) artifactResolverStrategy); - } - - /** - * Used by Serializers to lookup the schema for a given kafka record. - * - * @param topic - * @param headers, can be null - * @param data - * @param parsedSchema, can be null - * @return SchemaLookupResult - */ - @Deprecated - public SchemaLookupResult resolveSchema(String topic, Headers headers, DATA data, - ParsedSchema parsedSchema); - - /** - * Used by Deserializers to lookup the schema for a given kafka record. The schema resolver may use - * different pieces of information from the {@link ArtifactReference} depending on the configuration of - * the schema resolver. - * - * @param reference - * @return SchemaLookupResult - */ - @Deprecated - public SchemaLookupResult resolveSchemaByArtifactReference(ArtifactReference reference); - - /** - * Hard reset cache - */ - @Override - @Deprecated - public void reset(); - - /** - * @see io.apicurio.registry.resolver.SchemaResolver#configure(java.util.Map, - * io.apicurio.registry.resolver.SchemaParser) - */ - @Override - default void configure(Map configs, - io.apicurio.registry.resolver.SchemaParser schemaMapper) { - configure(configs, true, new SchemaParser() { - - /** - * @see io.apicurio.registry.serde.SchemaParser#artifactType() - */ - @Override - public String artifactType() { - return schemaMapper.artifactType(); - } - - /** - * @see io.apicurio.registry.serde.SchemaParser#parseSchema(byte[]) - */ - @Override - public Object parseSchema(byte[] rawSchema) { - return schemaMapper.parseSchema(rawSchema, Collections.emptyMap()); - } - - }); - } - - /** - * @see io.apicurio.registry.resolver.SchemaResolver#resolveSchema(io.apicurio.registry.resolver.data.Record) - */ - @Override - default io.apicurio.registry.resolver.SchemaLookupResult resolveSchema(Record data) { - KafkaSerdeRecord kdata = (KafkaSerdeRecord) data; - KafkaSerdeMetadata metadata = kdata.metadata(); - io.apicurio.registry.resolver.ParsedSchema ps = getSchemaParser().getSchemaFromData(data); - ParsedSchema compatps = ps == null ? null : new ParsedSchemaImpl() - .setParsedSchema(ps.getParsedSchema()).setRawSchema(ps.getRawSchema()); - return resolveSchema(metadata.getTopic(), metadata.getHeaders(), kdata.payload(), compatps) - .toCompat(); - } - - /** - * @see io.apicurio.registry.resolver.SchemaResolver#resolveSchemaByArtifactReference(io.apicurio.registry.resolver.strategy.ArtifactReference) - */ - @Override - default io.apicurio.registry.resolver.SchemaLookupResult resolveSchemaByArtifactReference( - io.apicurio.registry.resolver.strategy.ArtifactReference reference) { - return resolveSchemaByArtifactReference( - ArtifactReference.builder().contentId(reference.getContentId()) - .globalId(reference.getGlobalId()).groupId(reference.getGroupId()) - .artifactId(reference.getArtifactId()).version(reference.getVersion()).build()) - .toCompat(); - } - - /** - * @see java.io.Closeable#close() - */ - @Override - default void close() throws IOException { - } - -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaDeserializerConfig.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaDeserializerConfig.java deleted file mode 100644 index 91f8b07178..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaDeserializerConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -package io.apicurio.registry.serde.config; - -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; - -import java.util.Map; - -import static io.apicurio.registry.serde.SerdeConfig.FALLBACK_ARTIFACT_PROVIDER; -import static io.apicurio.registry.serde.SerdeConfig.FALLBACK_ARTIFACT_PROVIDER_DEFAULT; - -public class BaseKafkaDeserializerConfig extends BaseKafkaSerDeConfig { - - public static ConfigDef configDef() { - ConfigDef configDef = new ConfigDef().define(FALLBACK_ARTIFACT_PROVIDER, Type.CLASS, - FALLBACK_ARTIFACT_PROVIDER_DEFAULT, Importance.HIGH, "TODO docs"); - - return configDef; - } - - public BaseKafkaDeserializerConfig(Map originals) { - super(configDef(), originals); - } - - public Object getFallbackArtifactProvider() { - return this.get(FALLBACK_ARTIFACT_PROVIDER); - } - -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaSerDeConfig.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaSerDeConfig.java deleted file mode 100644 index 0d37fd372c..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/config/BaseKafkaSerDeConfig.java +++ /dev/null @@ -1,47 +0,0 @@ -package io.apicurio.registry.serde.config; - -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; - -import java.util.Map; - -import static io.apicurio.registry.serde.SerdeConfig.*; - -public class BaseKafkaSerDeConfig extends AbstractConfig { - - private static ConfigDef buildConfigDef(ConfigDef base) { - ConfigDef configDef = new ConfigDef(base) - .define(ID_HANDLER, Type.CLASS, ID_HANDLER_DEFAULT, Importance.MEDIUM, "TODO docs") - .define(ENABLE_HEADERS, Type.BOOLEAN, ENABLE_HEADERS_DEFAULT, Importance.MEDIUM, "TODO docs") - .define(HEADERS_HANDLER, Type.CLASS, HEADERS_HANDLER_DEFAULT, Importance.MEDIUM, "TODO docs") - .define(USE_ID, Type.STRING, USE_ID_DEFAULT, Importance.MEDIUM, "TODO docs"); - return configDef; - } - - public BaseKafkaSerDeConfig(ConfigDef configDef, Map originals) { - super(buildConfigDef(configDef), originals, false); - } - - public BaseKafkaSerDeConfig(Map originals) { - super(buildConfigDef(new ConfigDef()), originals, false); - } - - public Object getIdHandler() { - return this.get(ID_HANDLER); - } - - public boolean enableHeaders() { - return this.getBoolean(ENABLE_HEADERS); - } - - public Object getHeadersHandler() { - return this.get(HEADERS_HANDLER); - } - - public IdOption useIdOption() { - return IdOption.valueOf(this.getString(USE_ID)); - } - -} diff --git a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerConfig.java b/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerConfig.java deleted file mode 100644 index ad35eb6e22..0000000000 --- a/serdes/serde-common/src/main/java/io/apicurio/registry/serde/headers/DefaultHeadersHandlerConfig.java +++ /dev/null @@ -1,100 +0,0 @@ -package io.apicurio.registry.serde.headers; - -import io.apicurio.registry.serde.config.BaseKafkaSerDeConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigDef.Importance; -import org.apache.kafka.common.config.ConfigDef.Type; - -import java.util.Map; - -import static io.apicurio.registry.serde.SerdeConfig.*; -import static io.apicurio.registry.serde.SerdeHeaders.*; - -public class DefaultHeadersHandlerConfig extends BaseKafkaSerDeConfig { - - public static ConfigDef configDef() { - ConfigDef configDef = new ConfigDef() - .define(HEADER_KEY_GLOBAL_ID_OVERRIDE_NAME, Type.STRING, HEADER_KEY_GLOBAL_ID, - Importance.HIGH, "TODO docs") - .define(HEADER_KEY_CONTENT_ID_OVERRIDE_NAME, Type.STRING, HEADER_KEY_CONTENT_ID, - Importance.HIGH, "TODO docs") - .define(HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME, Type.STRING, HEADER_KEY_CONTENT_HASH, - Importance.HIGH, "TODO docs") - .define(HEADER_KEY_GROUP_ID_OVERRIDE_NAME, Type.STRING, HEADER_KEY_GROUP_ID, Importance.HIGH, - "TODO docs") - .define(HEADER_KEY_ARTIFACT_ID_OVERRIDE_NAME, Type.STRING, HEADER_KEY_ARTIFACT_ID, - Importance.HIGH, "TODO docs") - .define(HEADER_KEY_VERSION_OVERRIDE_NAME, Type.STRING, HEADER_KEY_VERSION, Importance.HIGH, - "TODO docs") - - .define(HEADER_VALUE_GLOBAL_ID_OVERRIDE_NAME, Type.STRING, HEADER_VALUE_GLOBAL_ID, - Importance.HIGH, "TODO docs") - .define(HEADER_VALUE_CONTENT_ID_OVERRIDE_NAME, Type.STRING, HEADER_VALUE_CONTENT_ID, - Importance.HIGH, "TODO docs") - .define(HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME, Type.STRING, HEADER_VALUE_CONTENT_HASH, - Importance.HIGH, "TODO docs") - .define(HEADER_VALUE_GROUP_ID_OVERRIDE_NAME, Type.STRING, HEADER_VALUE_GROUP_ID, - Importance.HIGH, "TODO docs") - .define(HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME, Type.STRING, HEADER_VALUE_ARTIFACT_ID, - Importance.HIGH, "TODO docs") - .define(HEADER_VALUE_VERSION_OVERRIDE_NAME, Type.STRING, HEADER_VALUE_VERSION, - Importance.HIGH, "TODO docs"); - - return configDef; - } - - public DefaultHeadersHandlerConfig(Map originals) { - super(configDef(), originals); - } - - public String getKeyGlobalIdHeader() { - return this.getString(HEADER_KEY_GLOBAL_ID_OVERRIDE_NAME); - } - - public String getKeyContentIdHeader() { - return this.getString(HEADER_KEY_CONTENT_ID_OVERRIDE_NAME); - } - - public String getKeyContentHashHeader() { - return this.getString(HEADER_KEY_CONTENT_HASH_OVERRIDE_NAME); - } - - public String getKeyGroupIdHeader() { - return this.getString(HEADER_KEY_GROUP_ID_OVERRIDE_NAME); - } - - public String getKeyArtifactIdHeader() { - return this.getString(HEADER_KEY_ARTIFACT_ID_OVERRIDE_NAME); - } - - public String getKeyVersionHeader() { - return this.getString(HEADER_KEY_VERSION_OVERRIDE_NAME); - } - - //// - - public String getValueGlobalIdHeader() { - return this.getString(HEADER_VALUE_GLOBAL_ID_OVERRIDE_NAME); - } - - public String getValueContentIdHeader() { - return this.getString(HEADER_VALUE_CONTENT_ID_OVERRIDE_NAME); - } - - public String getValueContentHashHeader() { - return this.getString(HEADER_VALUE_CONTENT_HASH_OVERRIDE_NAME); - } - - public String getValueGroupIdHeader() { - return this.getString(HEADER_VALUE_GROUP_ID_OVERRIDE_NAME); - } - - public String getValueArtifactIdHeader() { - return this.getString(HEADER_VALUE_ARTIFACT_ID_OVERRIDE_NAME); - } - - public String getValueVersionHeader() { - return this.getString(HEADER_VALUE_VERSION_OVERRIDE_NAME); - } - -} diff --git a/utils/converter/pom.xml b/utils/converter/pom.xml index e148c6b103..03fed91d45 100644 --- a/utils/converter/pom.xml +++ b/utils/converter/pom.xml @@ -20,12 +20,12 @@ io.apicurio - apicurio-registry-serdes-avro-serde + apicurio-registry-avro-serde-kafka io.apicurio - apicurio-registry-serdes-jsonschema-serde + apicurio-registry-jsonschema-serde-kafka diff --git a/utils/converter/src/main/java/io/apicurio/registry/utils/converter/ExtJsonConverter.java b/utils/converter/src/main/java/io/apicurio/registry/utils/converter/ExtJsonConverter.java index 9b7bd89409..8afd63ce12 100644 --- a/utils/converter/src/main/java/io/apicurio/registry/utils/converter/ExtJsonConverter.java +++ b/utils/converter/src/main/java/io/apicurio/registry/utils/converter/ExtJsonConverter.java @@ -130,7 +130,7 @@ public SchemaAndValue toConnectData(String topic, byte[] value) { } /** - * @see io.apicurio.registry.serde.SchemaParser#artifactType() + * @see io.apicurio.registry.resolver.SchemaParser#artifactType() */ @Override public String artifactType() { @@ -138,7 +138,8 @@ public String artifactType() { } /** - * @see io.apicurio.registry.serde.SchemaParser#parseSchema(byte[]) + * @see io.apicurio.registry.resolver.SchemaParser#parseSchema(byte[], Map>) */ @Override public JsonNode parseSchema(byte[] rawSchema, Map> resolvedReferences) { diff --git a/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/CompactFormatStrategy.java b/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/CompactFormatStrategy.java index 52b6cdcf3e..de7c33761f 100644 --- a/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/CompactFormatStrategy.java +++ b/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/CompactFormatStrategy.java @@ -2,6 +2,7 @@ import io.apicurio.registry.resolver.strategy.ArtifactReference; import io.apicurio.registry.serde.AbstractKafkaSerDe; +import io.apicurio.registry.serde.AbstractSerDe; import io.apicurio.registry.serde.Default4ByteIdHandler; import io.apicurio.registry.serde.IdHandler; @@ -27,7 +28,7 @@ public void setIdHandler(IdHandler idHandler) { @Override public byte[] fromConnectData(long contentId, byte[] bytes) { ByteBuffer buffer = ByteBuffer.allocate(1 + idHandler.idSize() + bytes.length); - buffer.put(AbstractKafkaSerDe.MAGIC_BYTE); + buffer.put(AbstractSerDe.MAGIC_BYTE); idHandler.writeId(ArtifactReference.fromContentId(contentId), buffer); buffer.put(bytes); return buffer.array(); diff --git a/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/JsonConverterRecord.java b/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/JsonConverterRecord.java index a8113465b0..e020e2e1fa 100644 --- a/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/JsonConverterRecord.java +++ b/utils/converter/src/main/java/io/apicurio/registry/utils/converter/json/JsonConverterRecord.java @@ -1,8 +1,8 @@ package io.apicurio.registry.utils.converter.json; -import io.apicurio.registry.serde.data.KafkaSerdeRecord; +import io.apicurio.registry.serde.data.SerdeRecord; -public class JsonConverterRecord extends KafkaSerdeRecord { +public class JsonConverterRecord extends SerdeRecord { /** * Constructor. @@ -15,7 +15,7 @@ public JsonConverterRecord(JsonConverterMetadata metadata, T payload) { } /** - * @see io.apicurio.registry.serde.data.KafkaSerdeRecord#metadata() + * @see io.apicurio.registry.serde.data.SerdeRecord#metadata() */ @Override public JsonConverterMetadata metadata() { diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/AbstractDirectoryParser.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/AbstractDirectoryParser.java index a212971001..29e7c02120 100644 --- a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/AbstractDirectoryParser.java +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/AbstractDirectoryParser.java @@ -10,6 +10,7 @@ import io.apicurio.registry.rest.client.models.CreateArtifactResponse; import io.apicurio.registry.rest.client.models.CreateVersion; import io.apicurio.registry.rest.client.models.IfArtifactExists; +import io.apicurio.registry.rest.client.models.ProblemDetails; import io.apicurio.registry.rest.client.models.VersionContent; import io.apicurio.registry.types.ContentTypes; import io.apicurio.registry.utils.IoUtil; @@ -120,16 +121,21 @@ private CreateArtifactResponse registerArtifact(RegisterArtifact artifact, Input }).collect(Collectors.toList())); createVersion.setContent(content); - var amd = client.groups().byGroupId(groupId).artifacts().post(createArtifact, config -> { - config.queryParameters.ifExists = IfArtifactExists.forValue(artifact.getIfExists().value()); - config.queryParameters.canonical = canonicalize; - }); + try { + var amd = client.groups().byGroupId(groupId).artifacts().post(createArtifact, config -> { + config.queryParameters.ifExists = IfArtifactExists.forValue(artifact.getIfExists().value()); + config.queryParameters.canonical = canonicalize; + }); + + // client.createArtifact(groupId, artifactId, version, type, ifExists, canonicalize, null, null, + // ContentTypes.APPLICATION_CREATE_EXTENDED, null, null, artifactContent, references); + log.info(String.format("Successfully registered artifact [%s] / [%s]. GlobalId is [%d]", groupId, + artifactId, amd.getVersion().getGlobalId())); - // client.createArtifact(groupId, artifactId, version, type, ifExists, canonicalize, null, null, - // ContentTypes.APPLICATION_CREATE_EXTENDED, null, null, artifactContent, references); - log.info(String.format("Successfully registered artifact [%s] / [%s]. GlobalId is [%d]", groupId, - artifactId, amd.getVersion().getGlobalId())); + return amd; - return amd; + } catch (ProblemDetails e) { + throw new RuntimeException(e.getDetail()); + } } } diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java index 35d6ec1f31..2077294087 100644 --- a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java @@ -14,6 +14,7 @@ import io.apicurio.registry.rest.client.models.CreateArtifactResponse; import io.apicurio.registry.rest.client.models.CreateVersion; import io.apicurio.registry.rest.client.models.IfArtifactExists; +import io.apicurio.registry.rest.client.models.ProblemDetails; import io.apicurio.registry.rest.client.models.VersionContent; import io.apicurio.registry.rest.client.models.VersionMetaData; import io.apicurio.registry.types.ArtifactType; @@ -137,21 +138,21 @@ protected void executeInternal() throws MojoExecutionException { registerWithAutoRefs(artifact, index, registrationStack); } else if (artifact.getAnalyzeDirectory() != null && artifact.getAnalyzeDirectory()) { // Auto - // register - // selected, - // we - // must - // figure - // out - // if - // the - // artifact - // has - // reference - // using - // the - // directory - // structure + // register + // selected, + // we + // must + // figure + // out + // if + // the + // artifact + // has + // reference + // using + // the + // directory + // structure registerDirectory(artifact); } else { @@ -321,17 +322,22 @@ private CreateArtifactResponse registerArtifact(RegisterArtifact artifact, Input }).collect(Collectors.toList())); createVersion.setContent(content); - var vmd = getClient().groups().byGroupId(groupId).artifacts().post(createArtifact, config -> { - if (artifact.getIfExists() != null) { - config.queryParameters.ifExists = IfArtifactExists.forValue(artifact.getIfExists().value()); - } - config.queryParameters.canonical = canonicalize; - }); + try { + var vmd = getClient().groups().byGroupId(groupId).artifacts().post(createArtifact, config -> { + if (artifact.getIfExists() != null) { + config.queryParameters.ifExists = IfArtifactExists + .forValue(artifact.getIfExists().value()); + } + config.queryParameters.canonical = canonicalize; + }); - getLog().info(String.format("Successfully registered artifact [%s] / [%s]. GlobalId is [%d]", - groupId, artifactId, vmd.getVersion().getGlobalId())); + getLog().info(String.format("Successfully registered artifact [%s] / [%s]. GlobalId is [%d]", + groupId, artifactId, vmd.getVersion().getGlobalId())); - return vmd; + return vmd; + } catch (ProblemDetails e) { + throw new RuntimeException(e.getDetail()); + } } private static boolean hasReferences(RegisterArtifact artifact) { @@ -376,7 +382,7 @@ private static ArtifactReference buildReferenceFromMetadata(VersionMetaData meta /** * Create a local index relative to the given file location. - * + * * @param file */ private static ReferenceIndex createIndex(File file) { @@ -437,7 +443,7 @@ private static File getLocalFile(Path path) { /** * Detects a loop by looking for the given artifact in the registration stack. - * + * * @param artifact * @param registrationStack */