This repository has been archived by the owner on Jun 7, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 292
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* the commit introduces Avro batch format for Nakadi Publishing API
- Loading branch information
Showing
69 changed files
with
1,032 additions
and
550 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
76 changes: 64 additions & 12 deletions
76
acceptance-test/src/acceptance-test/java/org/zalando/nakadi/webservice/BinaryEndToEndAT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,42 +1,94 @@ | ||
| package org.zalando.nakadi.webservice; | ||
|
|
||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import org.junit.Ignore; | ||
| import org.apache.avro.Schema; | ||
| import org.apache.avro.generic.GenericDatumWriter; | ||
| import org.apache.avro.generic.GenericRecordBuilder; | ||
| import org.apache.avro.io.EncoderFactory; | ||
| import org.junit.Assert; | ||
| import org.junit.Test; | ||
| import org.springframework.core.io.DefaultResourceLoader; | ||
| import org.zalando.nakadi.domain.EnrichmentStrategyDescriptor; | ||
| import org.zalando.nakadi.domain.EventCategory; | ||
| import org.zalando.nakadi.domain.EventTypeSchema; | ||
| import org.zalando.nakadi.domain.EventTypeSchemaBase; | ||
| import org.zalando.nakadi.domain.Subscription; | ||
| import org.zalando.nakadi.generated.avro.Envelope; | ||
| import org.zalando.nakadi.generated.avro.Metadata; | ||
| import org.zalando.nakadi.generated.avro.PublishingBatch; | ||
| import org.zalando.nakadi.utils.EventTypeTestBuilder; | ||
| import org.zalando.nakadi.utils.RandomSubscriptionBuilder; | ||
| import org.zalando.nakadi.utils.TestUtils; | ||
| import org.zalando.nakadi.webservice.utils.NakadiTestUtils; | ||
| import org.zalando.nakadi.webservice.utils.TestStreamingClient; | ||
|
|
||
| import java.util.Base64; | ||
| import java.io.ByteArrayOutputStream; | ||
| import java.io.IOException; | ||
| import java.nio.ByteBuffer; | ||
| import java.time.Instant; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| import static com.jayway.restassured.RestAssured.given; | ||
| import static org.zalando.nakadi.domain.SubscriptionBase.InitialPosition.BEGIN; | ||
| import static org.zalando.nakadi.webservice.utils.NakadiTestUtils.createSubscription; | ||
|
|
||
| public class BinaryEndToEndAT extends BaseAT { | ||
| private static final String TEST_ET_NAME = "nakadi.test-2022-05-06.et"; | ||
| private static final String TEST_DATA = "BAAAAGO0v/qJk2BIZjU5ZmVlNTQtMmNhYy00MTAzLWI4NTItOGMwOGRiZjhlNjEyAhJ0ZX" + | ||
| "N0LWZsb3cAAjECEnRlc3QtdXNlcjJuYWthZGkudGVzdC0yMDIyLTA1LTA2LmV0AAAAAAAAAABBCFBPU1QYL2V2ZW50LXR5cGVzAB50ZX" + | ||
| "N0LXVzZXItYWdlbnQMbmFrYWRpFGhhc2hlZC1hcHCSAxQCLQQtLfYBggU="; | ||
|
|
||
| @Test | ||
| @Ignore | ||
| public void testAvroPublishing() throws JsonProcessingException { | ||
| public void testAvroPublishingAndJsonConsumption() throws IOException { | ||
| final Schema schema = new Schema.Parser().parse(new DefaultResourceLoader() | ||
| .getResource("nakadi.end2end.avsc").getInputStream()); | ||
| final var et = EventTypeTestBuilder.builder() | ||
| .name(TEST_ET_NAME) | ||
| .category(EventCategory.BUSINESS) | ||
| .enrichmentStrategies(List.of(EnrichmentStrategyDescriptor.METADATA_ENRICHMENT)) | ||
| .schema(new EventTypeSchema(new EventTypeSchemaBase( | ||
| EventTypeSchemaBase.Type.AVRO_SCHEMA, | ||
| schema.toString()), "1.0.0", TestUtils.randomDate())) | ||
| .build(); | ||
| NakadiTestUtils.createEventTypeInNakadi(et); | ||
|
|
||
| final byte[] body = Base64.getDecoder().decode(TEST_DATA); | ||
| final ByteArrayOutputStream baos = new ByteArrayOutputStream(); | ||
| new GenericDatumWriter(schema).write( | ||
| new GenericRecordBuilder(schema).set("foo", "bar").build(), | ||
| EncoderFactory.get().directBinaryEncoder(baos, null)); | ||
|
|
||
| final PublishingBatch batch = PublishingBatch.newBuilder() | ||
| .setEvents(List.of(Envelope.newBuilder() | ||
| .setMetadata(Metadata.newBuilder() | ||
| .setEventType(TEST_ET_NAME) | ||
| .setVersion("1.0.0") | ||
| .setOccurredAt(Instant.now()) | ||
| .setEid("CE8C9EBC-3F19-4B9D-A453-08AD2EDA6028") | ||
| .build()) | ||
| .setPayload(ByteBuffer.wrap(baos.toByteArray())) | ||
| .build())) | ||
| .build(); | ||
|
|
||
| final ByteBuffer body = PublishingBatch.getEncoder().encode(batch); | ||
| final var response = given() | ||
| .contentType("application/avro-binary; charset=utf-8") | ||
| .body(body) | ||
| .contentType("application/avro-binary") | ||
| .body(body.array()) | ||
| .post(String.format("/event-types/%s/events", TEST_ET_NAME)); | ||
| response.print(); | ||
| response.then().statusCode(200); | ||
| // TODO add the consumption side once schema creation is done. | ||
|
|
||
| // check event is consumed and format is correct | ||
| final Subscription subscription = createSubscription( | ||
| RandomSubscriptionBuilder.builder() | ||
| .withEventType(TEST_ET_NAME) | ||
| .withStartFrom(BEGIN) | ||
| .buildSubscriptionBase()); | ||
| final TestStreamingClient client = TestStreamingClient.create(subscription.getId()).start(); | ||
|
|
||
| TestUtils.waitFor(() -> Assert.assertEquals(1, client.getBatches().size())); | ||
| final Map event = client.getBatches().get(0).getEvents().get(0); | ||
| Assert.assertEquals("bar", event.get("foo")); | ||
|
|
||
| final Map<String, Object> metadata = (Map<String, Object>) event.get("metadata"); | ||
| Assert.assertEquals("CE8C9EBC-3F19-4B9D-A453-08AD2EDA6028", metadata.get("eid")); | ||
| Assert.assertEquals("1.0.0", metadata.get("version")); | ||
| Assert.assertEquals(TEST_ET_NAME, metadata.get("event_type")); | ||
| } | ||
| } |
11 changes: 11 additions & 0 deletions
11
acceptance-test/src/acceptance-test/resources/nakadi.end2end.avsc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| { | ||
| "name": "TestEnd2End", | ||
| "namespace": "org.zalando.nakadi.generated.avro", | ||
| "type": "record", | ||
| "fields": [ | ||
| { | ||
| "name": "foo", | ||
| "type": "string" | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| rootProject.name = 'core-common' | ||
| rootProject.name = 'core-common' |
Oops, something went wrong.