Fixed bug with Nested Avro Composition

[sanyaivanov]

Overview

This contribution addresses an issue encountered with the avro-compose tool when attempting to parse schemas that involve nested composition. The specific error occurs due to an inability to redefine a type within the schema, causing the parsing process to fail.

Example of structure:

Error:

[ERROR] Error parsing files, remaining files with errors:
[ERROR] File Token.avsc is requiring type: null, parsing ended with exception.
org.apache.avro.SchemaParseException: Can't redefine: com.oivanov.examples.ExampleMetadata
	at org.apache.avro.Schema$Names.put(Schema.java:1604)
	at org.apache.avro.Schema$Names.add(Schema.java:1598)
	at org.apache.avro.Schema$Parser.addTypes(Schema.java:1394)
	at com.michalklempa.avro.compose.SchemaFile$Factory.parsed(SchemaFile.java:138)
	at com.michalklempa.avro.compose.Compose.compose(Compose.java:55)
	at com.michalklempa.avro.compose.Main.main(Main.java:146)
Exception in thread "main" java.lang.Exception: Error parsing files.
	at com.michalklempa.avro.compose.Compose.compose(Compose.java:89)
	at com.michalklempa.avro.compose.Main.main(Main.java:146)

Changes

• Schema Parsing: Updated approach adding schemas to the parser.

Implementation Details

Avro Schema:

{
  "type": "record",
  "name": "ExampleToken",
  "namespace": "com.oivanov.examples",
  "fields": [
    {
      "name": "raw",
      "type": "string"
    },
    {
      "name": "origin",
      "type": "string"
    },
    {
      "name": "metadataBasic",
      "type": "com.oivanov.examples.ExampleMetadata"
    },
    {
      "name": "metadata",
      "type": "com.oivanov.examples.ExampleMetadata"
    },
    {
      "name": "subscription",
      "type": "com.oivanov.examples.ExampleSubscription"
    }
  ]
}

Compiled Avro Schema:

{
  "type" : "record",
  "name" : "ExampleToken",
  "namespace" : "com.oivanov.examples",
  "fields" : [ {
    "name" : "raw",
    "type" : "string"
  }, {
    "name" : "origin",
    "type" : "string"
  }, {
    "name" : "metadataBasic",
    "type" : {
      "type" : "record",
      "name" : "ExampleMetadata",
      "fields" : [ {
        "name" : "timestamp",
        "type" : {
          "type" : "long",
          "logicalType" : "timestamp-millis"
        }
      }, {
        "name" : "source",
        "type" : "string"
      }, {
        "name" : "version",
        "type" : "string"
      } ]
    }
  }, {
    "name" : "metadata",
    "type" : "ExampleMetadata"
  }, {
    "name" : "subscription",
    "type" : {
      "type" : "record",
      "name" : "ExampleSubscription",
      "fields" : [ {
        "name" : "raw",
        "type" : "string"
      }, {
        "name" : "origin",
        "type" : [ "string", "null" ]
      }, {
        "name" : "mimetype",
        "type" : "string"
      }, {
        "name" : "metadata",
        "type" : "ExampleMetadata"
      } ]
    }
  } ]
}

Than it will be possible to generate Java POJO with nested fields:

package com.oivanov.examples;

import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.ByteBuffer;
import org.apache.avro.AvroMissingFieldException;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.data.RecordBuilder;
import org.apache.avro.data.TimeConversions;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.message.BinaryMessageDecoder;
import org.apache.avro.message.BinaryMessageEncoder;
import org.apache.avro.message.SchemaStore;
import org.apache.avro.specific.AvroGenerated;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecordBuilderBase;

@AvroGenerated
public class ExampleToken extends SpecificRecordBase implements SpecificRecord {
    private static final long serialVersionUID = -3067809783391727556L;
    public static final Schema SCHEMA$ = (new Schema.Parser()).parse("{\"type\":\"record\",\"name\":\"ExampleToken\",\"namespace\":\"com.oivanov.examples\",\"fields\":[{\"name\":\"raw\",\"type\":\"string\"},{\"name\":\"origin\",\"type\":\"string\"},{\"name\":\"metadataBasic\",\"type\":{\"type\":\"record\",\"name\":\"ExampleMetadata\",\"fields\":[{\"name\":\"timestamp\",\"type\":{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}},{\"name\":\"source\",\"type\":\"string\"},{\"name\":\"version\",\"type\":\"string\"}]}},{\"name\":\"metadata\",\"type\":\"ExampleMetadata\"},{\"name\":\"subscription\",\"type\":{\"type\":\"record\",\"name\":\"ExampleSubscription\",\"fields\":[{\"name\":\"raw\",\"type\":\"string\"},{\"name\":\"origin\",\"type\":[\"string\",\"null\"]},{\"name\":\"mimetype\",\"type\":\"string\"},{\"name\":\"metadata\",\"type\":\"ExampleMetadata\"}]}}]}");
    private static final SpecificData MODEL$ = new SpecificData();
    private static final BinaryMessageEncoder<ExampleToken> ENCODER;
    private static final BinaryMessageDecoder<ExampleToken> DECODER;
    private CharSequence raw;
    private CharSequence origin;
    private ExampleMetadata metadataBasic;
    private ExampleMetadata metadata;
    private ExampleSubscription subscription;
    private static final DatumWriter<ExampleToken> WRITER$;
    private static final DatumReader<ExampleToken> READER$;

    public static Schema getClassSchema() {
        return SCHEMA$;
    }

    public static BinaryMessageEncoder<ExampleToken> getEncoder() {
        return ENCODER;
    }

    public static BinaryMessageDecoder<ExampleToken> getDecoder() {
        return DECODER;
    }

    public static BinaryMessageDecoder<ExampleToken> createDecoder(SchemaStore resolver) {
        return new BinaryMessageDecoder(MODEL$, SCHEMA$, resolver);
    }

    public ByteBuffer toByteBuffer() throws IOException {
        return ENCODER.encode(this);
    }

    public static ExampleToken fromByteBuffer(ByteBuffer b) throws IOException {
        return (ExampleToken)DECODER.decode(b);
    }

    public ExampleToken() {
    }

    public ExampleToken(CharSequence raw, CharSequence origin, ExampleMetadata metadataBasic, ExampleMetadata metadata, ExampleSubscription subscription) {
        this.raw = raw;
        this.origin = origin;
        this.metadataBasic = metadataBasic;
        this.metadata = metadata;
        this.subscription = subscription;
    }

    public SpecificData getSpecificData() {
        return MODEL$;
    }

    public Schema getSchema() {
        return SCHEMA$;
    }

    public Object get(int field$) {
        switch (field$) {
            case 0 -> return this.raw;
            case 1 -> return this.origin;
            case 2 -> return this.metadataBasic;
            case 3 -> return this.metadata;
            case 4 -> return this.subscription;
            default -> throw new IndexOutOfBoundsException("Invalid index: " + field$);
        }
    }

    public void put(int field$, Object value$) {
        switch (field$) {
            case 0 -> this.raw = (CharSequence)value$;
            case 1 -> this.origin = (CharSequence)value$;
            case 2 -> this.metadataBasic = (ExampleMetadata)value$;
            case 3 -> this.metadata = (ExampleMetadata)value$;
            case 4 -> this.subscription = (ExampleSubscription)value$;
            default -> throw new IndexOutOfBoundsException("Invalid index: " + field$);
        }

    }

    public CharSequence getRaw() {
        return this.raw;
    }

    public void setRaw(CharSequence value) {
        this.raw = value;
    }

    public CharSequence getOrigin() {
        return this.origin;
    }

    public void setOrigin(CharSequence value) {
        this.origin = value;
    }

    public ExampleMetadata getMetadataBasic() {
        return this.metadataBasic;
    }

    public void setMetadataBasic(ExampleMetadata value) {
        this.metadataBasic = value;
    }

    public ExampleMetadata getMetadata() {
        return this.metadata;
    }

    public void setMetadata(ExampleMetadata value) {
        this.metadata = value;
    }

    public ExampleSubscription getSubscription() {
        return this.subscription;
    }

    public void setSubscription(ExampleSubscription value) {
        this.subscription = value;
    }

    public static Builder newBuilder() {
        return new Builder();
    }

    public static Builder newBuilder(Builder other) {
        return other == null ? new Builder() : new Builder(other);
    }

    public static Builder newBuilder(ExampleToken other) {
        return other == null ? new Builder() : new Builder(other);
    }

    public void writeExternal(ObjectOutput out) throws IOException {
        WRITER$.write(this, SpecificData.getEncoder(out));
    }

    public void readExternal(ObjectInput in) throws IOException {
        READER$.read(this, SpecificData.getDecoder(in));
    }

    static {
        MODEL$.addLogicalTypeConversion(new TimeConversions.TimestampMillisConversion());
        ENCODER = new BinaryMessageEncoder(MODEL$, SCHEMA$);
        DECODER = new BinaryMessageDecoder(MODEL$, SCHEMA$);
        WRITER$ = MODEL$.createDatumWriter(SCHEMA$);
        READER$ = MODEL$.createDatumReader(SCHEMA$);
    }

    @AvroGenerated
    public static class Builder extends SpecificRecordBuilderBase<ExampleToken> implements RecordBuilder<ExampleToken> {
        private CharSequence raw;
        private CharSequence origin;
        private ExampleMetadata metadataBasic;
        private ExampleMetadata.Builder metadataBasicBuilder;
        private ExampleMetadata metadata;
        private ExampleMetadata.Builder metadataBuilder;
        private ExampleSubscription subscription;
        private ExampleSubscription.Builder subscriptionBuilder;

        private Builder() {
            super(ExampleToken.SCHEMA$, ExampleToken.MODEL$);
        }

        private Builder(Builder other) {
            super(other);
            if (isValidValue(this.fields()[0], other.raw)) {
                this.raw = (CharSequence)this.data().deepCopy(this.fields()[0].schema(), other.raw);
                this.fieldSetFlags()[0] = other.fieldSetFlags()[0];
            }

            if (isValidValue(this.fields()[1], other.origin)) {
                this.origin = (CharSequence)this.data().deepCopy(this.fields()[1].schema(), other.origin);
                this.fieldSetFlags()[1] = other.fieldSetFlags()[1];
            }

            if (isValidValue(this.fields()[2], other.metadataBasic)) {
                this.metadataBasic = (ExampleMetadata)this.data().deepCopy(this.fields()[2].schema(), other.metadataBasic);
                this.fieldSetFlags()[2] = other.fieldSetFlags()[2];
            }

            if (other.hasMetadataBasicBuilder()) {
                this.metadataBasicBuilder = ExampleMetadata.newBuilder(other.getMetadataBasicBuilder());
            }

            if (isValidValue(this.fields()[3], other.metadata)) {
                this.metadata = (ExampleMetadata)this.data().deepCopy(this.fields()[3].schema(), other.metadata);
                this.fieldSetFlags()[3] = other.fieldSetFlags()[3];
            }

            if (other.hasMetadataBuilder()) {
                this.metadataBuilder = ExampleMetadata.newBuilder(other.getMetadataBuilder());
            }

            if (isValidValue(this.fields()[4], other.subscription)) {
                this.subscription = (ExampleSubscription)this.data().deepCopy(this.fields()[4].schema(), other.subscription);
                this.fieldSetFlags()[4] = other.fieldSetFlags()[4];
            }

            if (other.hasSubscriptionBuilder()) {
                this.subscriptionBuilder = ExampleSubscription.newBuilder(other.getSubscriptionBuilder());
            }

        }

        private Builder(ExampleToken other) {
            super(ExampleToken.SCHEMA$, ExampleToken.MODEL$);
            if (isValidValue(this.fields()[0], other.raw)) {
                this.raw = (CharSequence)this.data().deepCopy(this.fields()[0].schema(), other.raw);
                this.fieldSetFlags()[0] = true;
            }

            if (isValidValue(this.fields()[1], other.origin)) {
                this.origin = (CharSequence)this.data().deepCopy(this.fields()[1].schema(), other.origin);
                this.fieldSetFlags()[1] = true;
            }

            if (isValidValue(this.fields()[2], other.metadataBasic)) {
                this.metadataBasic = (ExampleMetadata)this.data().deepCopy(this.fields()[2].schema(), other.metadataBasic);
                this.fieldSetFlags()[2] = true;
            }

            this.metadataBasicBuilder = null;
            if (isValidValue(this.fields()[3], other.metadata)) {
                this.metadata = (ExampleMetadata)this.data().deepCopy(this.fields()[3].schema(), other.metadata);
                this.fieldSetFlags()[3] = true;
            }

            this.metadataBuilder = null;
            if (isValidValue(this.fields()[4], other.subscription)) {
                this.subscription = (ExampleSubscription)this.data().deepCopy(this.fields()[4].schema(), other.subscription);
                this.fieldSetFlags()[4] = true;
            }

            this.subscriptionBuilder = null;
        }

        public CharSequence getRaw() {
            return this.raw;
        }

        public Builder setRaw(CharSequence value) {
            this.validate(this.fields()[0], value);
            this.raw = value;
            this.fieldSetFlags()[0] = true;
            return this;
        }

        public boolean hasRaw() {
            return this.fieldSetFlags()[0];
        }

        public Builder clearRaw() {
            this.raw = null;
            this.fieldSetFlags()[0] = false;
            return this;
        }

        public CharSequence getOrigin() {
            return this.origin;
        }

        public Builder setOrigin(CharSequence value) {
            this.validate(this.fields()[1], value);
            this.origin = value;
            this.fieldSetFlags()[1] = true;
            return this;
        }

        public boolean hasOrigin() {
            return this.fieldSetFlags()[1];
        }

        public Builder clearOrigin() {
            this.origin = null;
            this.fieldSetFlags()[1] = false;
            return this;
        }

        public ExampleMetadata getMetadataBasic() {
            return this.metadataBasic;
        }

        public Builder setMetadataBasic(ExampleMetadata value) {
            this.validate(this.fields()[2], value);
            this.metadataBasicBuilder = null;
            this.metadataBasic = value;
            this.fieldSetFlags()[2] = true;
            return this;
        }

        public boolean hasMetadataBasic() {
            return this.fieldSetFlags()[2];
        }

        public ExampleMetadata.Builder getMetadataBasicBuilder() {
            if (this.metadataBasicBuilder == null) {
                if (this.hasMetadataBasic()) {
                    this.setMetadataBasicBuilder(ExampleMetadata.newBuilder(this.metadataBasic));
                } else {
                    this.setMetadataBasicBuilder(ExampleMetadata.newBuilder());
                }
            }

            return this.metadataBasicBuilder;
        }

        public Builder setMetadataBasicBuilder(ExampleMetadata.Builder value) {
            this.clearMetadataBasic();
            this.metadataBasicBuilder = value;
            return this;
        }

        public boolean hasMetadataBasicBuilder() {
            return this.metadataBasicBuilder != null;
        }

        public Builder clearMetadataBasic() {
            this.metadataBasic = null;
            this.metadataBasicBuilder = null;
            this.fieldSetFlags()[2] = false;
            return this;
        }

        public ExampleMetadata getMetadata() {
            return this.metadata;
        }

        public Builder setMetadata(ExampleMetadata value) {
            this.validate(this.fields()[3], value);
            this.metadataBuilder = null;
            this.metadata = value;
            this.fieldSetFlags()[3] = true;
            return this;
        }

        public boolean hasMetadata() {
            return this.fieldSetFlags()[3];
        }

        public ExampleMetadata.Builder getMetadataBuilder() {
            if (this.metadataBuilder == null) {
                if (this.hasMetadata()) {
                    this.setMetadataBuilder(ExampleMetadata.newBuilder(this.metadata));
                } else {
                    this.setMetadataBuilder(ExampleMetadata.newBuilder());
                }
            }

            return this.metadataBuilder;
        }

        public Builder setMetadataBuilder(ExampleMetadata.Builder value) {
            this.clearMetadata();
            this.metadataBuilder = value;
            return this;
        }

        public boolean hasMetadataBuilder() {
            return this.metadataBuilder != null;
        }

        public Builder clearMetadata() {
            this.metadata = null;
            this.metadataBuilder = null;
            this.fieldSetFlags()[3] = false;
            return this;
        }

        public ExampleSubscription getSubscription() {
            return this.subscription;
        }

        public Builder setSubscription(ExampleSubscription value) {
            this.validate(this.fields()[4], value);
            this.subscriptionBuilder = null;
            this.subscription = value;
            this.fieldSetFlags()[4] = true;
            return this;
        }

        public boolean hasSubscription() {
            return this.fieldSetFlags()[4];
        }

        public ExampleSubscription.Builder getSubscriptionBuilder() {
            if (this.subscriptionBuilder == null) {
                if (this.hasSubscription()) {
                    this.setSubscriptionBuilder(ExampleSubscription.newBuilder(this.subscription));
                } else {
                    this.setSubscriptionBuilder(ExampleSubscription.newBuilder());
                }
            }

            return this.subscriptionBuilder;
        }

        public Builder setSubscriptionBuilder(ExampleSubscription.Builder value) {
            this.clearSubscription();
            this.subscriptionBuilder = value;
            return this;
        }

        public boolean hasSubscriptionBuilder() {
            return this.subscriptionBuilder != null;
        }

        public Builder clearSubscription() {
            this.subscription = null;
            this.subscriptionBuilder = null;
            this.fieldSetFlags()[4] = false;
            return this;
        }

        public ExampleToken build() {
            try {
                ExampleToken record = new ExampleToken();
                record.raw = this.fieldSetFlags()[0] ? this.raw : (CharSequence)this.defaultValue(this.fields()[0]);
                record.origin = this.fieldSetFlags()[1] ? this.origin : (CharSequence)this.defaultValue(this.fields()[1]);
                AvroMissingFieldException e;
                if (this.metadataBasicBuilder != null) {
                    try {
                        record.metadataBasic = this.metadataBasicBuilder.build();
                    } catch (AvroMissingFieldException var5) {
                        e = var5;
                        e.addParentField(record.getSchema().getField("metadataBasic"));
                        throw e;
                    }
                } else {
                    record.metadataBasic = this.fieldSetFlags()[2] ? this.metadataBasic : (ExampleMetadata)this.defaultValue(this.fields()[2]);
                }

                if (this.metadataBuilder != null) {
                    try {
                        record.metadata = this.metadataBuilder.build();
                    } catch (AvroMissingFieldException var4) {
                        e = var4;
                        e.addParentField(record.getSchema().getField("metadata"));
                        throw e;
                    }
                } else {
                    record.metadata = this.fieldSetFlags()[3] ? this.metadata : (ExampleMetadata)this.defaultValue(this.fields()[3]);
                }

                if (this.subscriptionBuilder != null) {
                    try {
                        record.subscription = this.subscriptionBuilder.build();
                    } catch (AvroMissingFieldException var3) {
                        e = var3;
                        e.addParentField(record.getSchema().getField("subscription"));
                        throw e;
                    }
                } else {
                    record.subscription = this.fieldSetFlags()[4] ? this.subscription : (ExampleSubscription)this.defaultValue(this.fields()[4]);
                }

                return record;
            } catch (AvroMissingFieldException var6) {
                AvroMissingFieldException e = var6;
                throw e;
            } catch (Exception var7) {
                Exception e = var7;
                throw new AvroRuntimeException(e);
            }
        }
    }
}

Feel free to ping me if you have questions.
Cheers 👋