diff --git a/inlong-sort/sort-formats/format-common/pom.xml b/inlong-sort/sort-formats/format-common/pom.xml index f59734a64b1..31fa4137a59 100644 --- a/inlong-sort/sort-formats/format-common/pom.xml +++ b/inlong-sort/sort-formats/format-common/pom.xml @@ -31,14 +31,110 @@ ${project.parent.parent.parent.basedir} + 2.5.0 + + org.apache.inlong + inlong-common + ${project.version} + org.apache.flink flink-shaded-jackson ${flink.jackson.version} provided + + com.google.protobuf + protobuf-java + ${binlog.protobuf.version} + jar + provided + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf.plugin.version} + true + + com.google.protobuf:protoc:2.5.0:exe:${os.detected.classifier} + ${project.basedir}/src/main/proto + ${project.basedir}/src/main/java + false + + + + + compile + + compile + + + + + com.mycila + license-maven-plugin + 4.1 + + + +
${project.basedir}/src/main/HEADER
+ + **/*.properties + *.sh + *.yml + .editorconfig + .gitignore + **/*.md + **/*.xml + src/main/java/org/apache/inlong/sort/formats/base/*.java + src/main/java/org/apache/inlong/sort/formats/common/*.java + src/main/java/org/apache/inlong/sort/formats/inlongmsg/*.java + src/main/java/org/apache/inlong/sort/formats/util/*.java + + + + + SLASHSTAR_STYLE + JAVADOC_STYLE + DOUBLESLASH_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + JAVADOC_STYLE + XML_STYLE + SCRIPT_STYLE + JAVADOC_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + SCRIPT_STYLE + + + + + + format + + compile + + + + + + + kr.motd.maven + os-maven-plugin + ${os.plugin.version} + + + diff --git a/inlong-sort/sort-formats/format-common/src/main/HEADER b/inlong-sort/sort-formats/format-common/src/main/HEADER new file mode 100644 index 00000000000..176804ebfb5 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/HEADER @@ -0,0 +1,14 @@ +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptor.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptor.java similarity index 100% rename from inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptor.java rename to inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptor.java diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptorValidator.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptorValidator.java similarity index 100% rename from inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptorValidator.java rename to inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/FormatDescriptorValidator.java diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatConstants.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatConstants.java new file mode 100644 index 00000000000..42b367c2fbb --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatConstants.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.base; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; + +/** + * Common constants used in various formats. + */ +public class TableFormatConstants { + + public static final String FORMAT_SCHEMA = "format.schema"; + public static final String FORMAT_DELIMITER = "format.delimiter"; + public static final String FORMAT_DERIVE_SCHEMA = "format.derive-schema"; + public static final String FORMAT_ENTRY_DELIMITER = "format.entry-delimiter"; + public static final String FORMAT_KV_DELIMITER = "format.kv-delimiter"; + public static final String FORMAT_LINE_DELIMITER = "format.line-delimiter"; + public static final String FORMAT_NULL_LITERAL = "format.null-literal"; + public static final String FORMAT_ESCAPE_CHARACTER = "format.escape-character"; + public static final String FORMAT_QUOTE_CHARACTER = "format.quote-character"; + public static final String FORMAT_IGNORE_ERRORS = "format.ignore-errors"; + public static final String FORMAT_CHARSET = "format.charset"; + public static final String FORMAT_TYPE = "format.type"; + public static final String FORMAT_PROPERTY_VERSION = "format.property-version"; + public static final String FORMAT_FIELD_DELIMITER = "format.field-delimiter"; + public static final String FORMAT_TIME_FIELD_NAME = "format.time-field-name"; + public static final String FORMAT_KV_ENTRY_DELIMITER = "entry-delimiter"; + public static final String FORMAT_ATTRIBUTE_FIELD_NAME = "format.attribute-field-name"; + public static final String FORMAT_IS_MIXED = "format.is-mixed"; + public static final String FORMAT_DELETE_HEAD_DELIMITER = "format.delete-head-delimiter"; + public static final String FORMAT_RETAIN_PREDEFINED_FIELD = "format.retain-predefined-field"; + public static final String FORMAT_METADATA_FIELD_NAME = "format.metadata-field-name"; + public static final String FORMAT_INCLUDE_UPDATE_BEFORE = "format.include-update-before"; + public static final String FORMAT_PROPERTIES = "properties"; + + public static final char DEFAULT_DELIMITER = ','; + public static final char DEFAULT_ENTRY_DELIMITER = '&'; + public static final char DEFAULT_KV_DELIMITER = '='; + public static final Character DEFAULT_LINE_DELIMITER = null; + public static final Character DEFAULT_ESCAPE_CHARACTER = null; + public static final Character DEFAULT_QUOTE_CHARACTER = null; + public static final String DEFAULT_NULL_LITERAL = null; + public static final boolean DEFAULT_IGNORE_ERRORS = false; + public static final String DEFAULT_CHARSET = "UTF-8"; + + public static final boolean DEFAULT_IS_MIXED = false; + public static final boolean DEFAULT_DELETE_HEAD_DELIMITER = false; + public static final boolean DEFAULT_RETAIN_PREDEFINED_FIELD = true; + public static final int DEFAULT_BYTE_ARRAY_STREAM_LENGTH = 2048; + public static final String DEFAULT_METADATA_FIELD_NAME = "inlongmsg_metadata"; + + public static final boolean DEFAULT_INCLUDE_UPDATE_BEFORE = false; + + public static final DateTimeFormatter SQL_TIME_FORMAT = + new DateTimeFormatterBuilder() + .appendPattern("HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(); + + public static final DateTimeFormatter SQL_TIMESTAMP_FORMAT = + new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .appendLiteral(' ') + .append(SQL_TIME_FORMAT) + .toFormatter(); + + public static final DateTimeFormatter ISO8601_TIMESTAMP_FORMAT = + DateTimeFormatter.ISO_LOCAL_DATE_TIME; + + public static final DateTimeFormatter SQL_TIMESTAMP_WITH_LOCAL_TIMEZONE_FORMAT = + new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .appendLiteral(' ') + .append(SQL_TIME_FORMAT) + .appendPattern("'Z'") + .toFormatter(); + + public static final DateTimeFormatter ISO8601_TIMESTAMP_WITH_LOCAL_TIMEZONE_FORMAT = + new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .appendLiteral('T') + .append(DateTimeFormatter.ISO_LOCAL_TIME) + .appendPattern("'Z'") + .toFormatter(); +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatOptions.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatOptions.java new file mode 100644 index 00000000000..aa05bfa379f --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatOptions.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.base; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +/** + * Options for table formats. + */ +public class TableFormatOptions { + + public static final ConfigOption IGNORE_ERRORS = + ConfigOptions.key(TableFormatConstants.FORMAT_IGNORE_ERRORS) + .booleanType() + .defaultValue(TableFormatConstants.DEFAULT_IGNORE_ERRORS); + + public static final ConfigOption ROW_FORMAT_INFO = + ConfigOptions.key("row.format.info") + .stringType() + .noDefaultValue(); +} diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatUtils.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatUtils.java similarity index 83% rename from inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatUtils.java rename to inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatUtils.java index 5b9463c4a1f..d83108de396 100644 --- a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatUtils.java +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TableFormatUtils.java @@ -63,6 +63,7 @@ import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.descriptors.DescriptorProperties; @@ -92,9 +93,12 @@ import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.types.Row; +import java.lang.reflect.Array; +import java.math.BigDecimal; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.IntStream; import static org.apache.flink.table.factories.TableFormatFactoryBase.deriveSchema; import static org.apache.flink.util.Preconditions.checkState; @@ -151,95 +155,6 @@ public static SerializationSchema getSerializationSchema( return serializationSchemaFactory.createSerializationSchema(properties); } - /** - * Returns the {@link DeserializationSchema} described by the given - * properties. - * - * @param properties The properties describing the deserializer. - * @param fields The fields to project. - * @param classLoader The class loader for the deserializer. - * @param The type of the data. - * @return The {@link DeserializationSchema} described by the properties. - */ - public static DeserializationSchema getProjectedDeserializationSchema( - final Map properties, - final int[] fields, - final ClassLoader classLoader) { - final ProjectedDeserializationSchemaFactory deserializationSchemaFactory = - TableFactoryService.find( - ProjectedDeserializationSchemaFactory.class, - properties, - classLoader); - - return deserializationSchemaFactory - .createProjectedDeserializationSchema(properties, fields); - } - - /** - * Returns the {@link SerializationSchema} described by the given - * properties. - * - * @param properties The properties describing the serializer. - * @param fields The fields to project. - * @param classLoader The class loader for the serializer. - * @return The {@link SerializationSchema} described by the properties. - */ - public static SerializationSchema getProjectedSerializationSchema( - final Map properties, - final int[] fields, - final ClassLoader classLoader) { - final ProjectedSerializationSchemaFactory serializationSchemaFactory = - TableFactoryService.find( - ProjectedSerializationSchemaFactory.class, - properties, - classLoader); - - return serializationSchemaFactory - .createProjectedSerializationSchema(properties, fields); - } - - /** - * Returns the {@link TableFormatSerializer} described by the given - * properties. - * - * @param properties The properties describing the serializer. - * @param classLoader The class loader for the serializer. - * @return The {@link TableFormatSerializer} described by the properties. - */ - public static TableFormatSerializer getTableFormatSerializer( - final Map properties, - final ClassLoader classLoader) { - final TableFormatSerializerFactory tableFormatSerializerFactory = - TableFactoryService.find( - TableFormatSerializerFactory.class, - properties, - classLoader); - - return tableFormatSerializerFactory - .createFormatSerializer(properties); - } - - /** - * Returns the {@link TableFormatDeserializer} described by the - * given properties. - * - * @param properties The properties describing the deserializer. - * @param classLoader The class loader for the deserializer. - * @return The {@link TableFormatDeserializer} described by the properties. - */ - public static TableFormatDeserializer getTableFormatDeserializer( - final Map properties, - final ClassLoader classLoader) { - final TableFormatDeserializerFactory tableFormatDeserializerFactory = - TableFactoryService.find( - TableFormatDeserializerFactory.class, - properties, - classLoader); - - return tableFormatDeserializerFactory - .createFormatDeserializer(properties); - } - /** * Derive the format information for the given type. * @@ -448,6 +363,73 @@ public static TypeInformation getType(TypeInfo typeInfo) { } } + /** + * Returns the Flink SQL type represented by the given format. + * Please see {@link org.apache.flink.table.types.utils.TypeInfoDataTypeConverter} and + * {@link org.apache.flink.table.types.utils.LegacyTypeInfoDataTypeConverter}. + * + * @param typeInfo The type information of flink formats. + * @return The Flink SQL data type represented by the given format. + */ + public static DataType getDataType(TypeInfo typeInfo) { + if (typeInfo instanceof StringTypeInfo) { + return DataTypes.STRING().bridgedTo(String.class); + } else if (typeInfo instanceof BooleanTypeInfo) { + return DataTypes.BOOLEAN().bridgedTo(Boolean.class); + } else if (typeInfo instanceof ByteTypeInfo) { + return DataTypes.TINYINT().bridgedTo(Byte.class); + } else if (typeInfo instanceof ShortTypeInfo) { + return DataTypes.SMALLINT().bridgedTo(Short.class); + } else if (typeInfo instanceof IntTypeInfo) { + return DataTypes.INT().bridgedTo(Integer.class); + } else if (typeInfo instanceof LongTypeInfo) { + return DataTypes.BIGINT().bridgedTo(Long.class); + } else if (typeInfo instanceof FloatTypeInfo) { + return DataTypes.FLOAT().bridgedTo(Float.class); + } else if (typeInfo instanceof DoubleTypeInfo) { + return DataTypes.DOUBLE().bridgedTo(Double.class); + } else if (typeInfo instanceof DecimalTypeInfo) { + return DataTypes.DECIMAL(38, 18).bridgedTo(BigDecimal.class); + } else if (typeInfo instanceof DateTypeInfo) { + return DataTypes.DATE().bridgedTo(java.sql.Date.class); + } else if (typeInfo instanceof TimeTypeInfo) { + return DataTypes.TIME(0).bridgedTo(java.sql.Time.class); + } else if (typeInfo instanceof TimestampTypeInfo) { + return DataTypes.TIMESTAMP(3).bridgedTo(java.sql.Timestamp.class); + } else if (typeInfo instanceof ArrayTypeInfo) { + ArrayTypeInfo arrayTypeInfo = (ArrayTypeInfo) typeInfo; + TypeInfo elementTypeInfo = + arrayTypeInfo.getElementTypeInfo(); + DataType elementType = getDataType(elementTypeInfo); + // copied from ObjectArrayTypeInfo#getInfoFor + Class arrayClass = Array.newInstance(elementType.getConversionClass(), 0).getClass(); + return DataTypes.ARRAY(elementType).bridgedTo(arrayClass); + } else if (typeInfo instanceof MapTypeInfo) { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + TypeInfo keyTypeInfo = mapTypeInfo.getKeyTypeInfo(); + TypeInfo valueTypeInfo = mapTypeInfo.getValueTypeInfo(); + + DataType keyType = getDataType(keyTypeInfo); + DataType valueType = getDataType(valueTypeInfo); + + return DataTypes.MAP(keyType, valueType).bridgedTo(Map.class); + } else if (typeInfo instanceof RowTypeInfo) { + RowTypeInfo rowTypeInfo = (RowTypeInfo) typeInfo; + String[] fieldNames = rowTypeInfo.getFieldNames(); + TypeInfo[] fieldTypeInfos = rowTypeInfo.getFieldTypeInfos(); + + DataTypes.Field[] fields = IntStream.range(0, fieldNames.length) + .mapToObj(i -> { + DataType fieldType = getDataType(fieldTypeInfos[i]); + return DataTypes.FIELD(fieldNames[i], fieldType); + }).toArray(DataTypes.Field[]::new); + + return DataTypes.ROW(fields).bridgedTo(Row.class); + } else { + throw new IllegalStateException("Unexpected format."); + } + } + /** * Returns the format defined in the given property. * @@ -613,6 +595,31 @@ public static String serializeBasicField( } } + public static RowFormatInfo deriveRowFormatInfo( + RowType rowType) { + List fields = rowType.getFields(); + FormatInfo[] fieldFormatInfos = new FormatInfo[fields.size()]; + for (int i = 0; i < fields.size(); ++i) { + LogicalType fieldType = fields.get(i).getType(); + fieldFormatInfos[i] = TableFormatUtils.deriveFormatInfo(fieldType); + } + + return new RowFormatInfo(rowType.getFieldNames().toArray(new String[1]), fieldFormatInfos); + } + + public static RowFormatInfo deserializeRowFormatInfo(String rowFormatInfoStr) { + try { + FormatInfo formatInfo = FormatUtils.demarshall(rowFormatInfoStr); + if (!(formatInfo instanceof RowFormatInfo)) { + throw new IllegalStateException("Unexpected format type."); + } + + return (RowFormatInfo) formatInfo; + } catch (Exception e) { + throw new ValidationException("The schema is invalid.", e); + } + } + public static void getValidateProperties(DescriptorProperties properties) { properties.validateString(TableFormatConstants.FORMAT_TYPE, false, 1); properties.validateString(FORMAT_PROPERTY_VERSION, true, 1); diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TextFormatBuilder.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatBuilder.java similarity index 100% rename from inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TextFormatBuilder.java rename to inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatBuilder.java diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptor.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptor.java similarity index 100% rename from inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptor.java rename to inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptor.java diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptorValidator.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptorValidator.java similarity index 100% rename from inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptorValidator.java rename to inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatDescriptorValidator.java diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatOptions.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatOptions.java new file mode 100644 index 00000000000..ad519a334bc --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatOptions.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.base; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_CHARSET; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_ESCAPE_CHARACTER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_QUOTE_CHARACTER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_CHARSET; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_ESCAPE_CHARACTER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_FIELD_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_NULL_LITERAL; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_QUOTE_CHARACTER; + +/** + * Text format options. + */ +@PublicEvolving +public class TextFormatOptions { + + public static final ConfigOption CHARSET = + ConfigOptions.key(FORMAT_CHARSET) + .stringType() + .defaultValue(DEFAULT_CHARSET) + .withDescription("Optional text encoding format ('utf-8' by default)"); + + public static final ConfigOption FIELD_DELIMITER = + ConfigOptions.key(FORMAT_FIELD_DELIMITER) + .stringType() + .defaultValue(String.valueOf(DEFAULT_DELIMITER)) + .withDescription("Optional field delimiter character (',' by default)"); + + public static final ConfigOption QUOTE_CHARACTER = + ConfigOptions.key(FORMAT_QUOTE_CHARACTER) + .stringType() + .defaultValue(String.valueOf(DEFAULT_QUOTE_CHARACTER)) + .withDescription( + "Optional quote character for enclosing field values ('\"' by default)"); + + public static final ConfigOption ESCAPE_CHARACTER = + ConfigOptions.key(FORMAT_ESCAPE_CHARACTER) + .stringType() + .defaultValue(String.valueOf(DEFAULT_ESCAPE_CHARACTER)) + .withDescription( + "Optional escape character for escaping values (disabled by default)"); + + public static final ConfigOption NULL_LITERAL = + ConfigOptions.key(FORMAT_NULL_LITERAL) + .stringType() + .defaultValue(FORMAT_NULL_LITERAL) + .withDescription( + "Optional null literal string that is interpreted as a\n" + + "null value (disabled by default)"); + + public static final ConfigOption KV_ENTRY_DELIMITER = + ConfigOptions.key(TableFormatConstants.FORMAT_KV_ENTRY_DELIMITER) + .stringType() + .defaultValue(String.valueOf(TableFormatConstants.DEFAULT_ENTRY_DELIMITER)); + + public static final ConfigOption KV_DELIMITER = + ConfigOptions.key(TableFormatConstants.FORMAT_KV_DELIMITER) + .stringType() + .defaultValue(String.valueOf(TableFormatConstants.DEFAULT_KV_DELIMITER)); + + public static final ConfigOption LINE_DELIMITER = + ConfigOptions.key(TableFormatConstants.FORMAT_LINE_DELIMITER) + .stringType() + .defaultValue(String.valueOf(TableFormatConstants.DEFAULT_LINE_DELIMITER)); + + public static final ConfigOption FORMAT_PROPERTIES = + ConfigOptions.key(TableFormatConstants.FORMAT_PROPERTIES) + .stringType() + .noDefaultValue(); + + public static final ConfigOption MAP_NULL_KEY_MODE = + ConfigOptions.key("map-null-key.mode") + .stringType() + .defaultValue("FAIL") + .withDescription( + "Optional flag to control the handling mode when serializing null key for map data, FAIL by default." + + " Option DROP will drop null key entries for map data." + + " Option LITERAL will use 'map-null-key.literal' as key literal."); + + public static final ConfigOption FAIL_ON_MISSING_FIELD = + ConfigOptions.key("fail-on-missing-field") + .booleanType() + .defaultValue(false) + .withDescription( + "Optional flag to specify whether to fail if a field is missing or not, false by default."); + + public static final ConfigOption MAP_NULL_KEY_LITERAL = + ConfigOptions.key("map-null-key.literal") + .stringType() + .defaultValue("null") + .withDescription( + "Optional flag to specify string literal for null keys when 'map-null-key.mode' is LITERAL, \"null\" by default."); + + public static final ConfigOption TIMESTAMP_FORMAT = + ConfigOptions.key("timestamp-format.standard") + .stringType() + .defaultValue("SQL") + .withDescription( + "Optional flag to specify timestamp format, SQL by default." + + " Option ISO-8601 will parse input timestamp in \"yyyy-MM-ddTHH:mm:ss.s{precision}\" format and output timestamp in the same format." + + " Option SQL will parse input timestamp in \"yyyy-MM-dd HH:mm:ss.s{precision}\" format and output timestamp in the same format."); + + /** Handling mode for map data with null key. */ + public enum MapNullKeyMode { + FAIL, + DROP, + LITERAL + } + + /** Timestamp format Enums. */ + public enum TimestampFormat { + /** + * Options to specify TIMESTAMP/TIMESTAMP_WITH_LOCAL_ZONE format. It will parse TIMESTAMP in + * "yyyy-MM-dd HH:mm:ss.s{precision}" format, TIMESTAMP_WITH_LOCAL_TIMEZONE in "yyyy-MM-dd + * HH:mm:ss.s{precision}'Z'" and output in the same format. + */ + SQL, + + /** + * Options to specify TIMESTAMP/TIMESTAMP_WITH_LOCAL_ZONE format. It will parse TIMESTAMP in + * "yyyy-MM-ddTHH:mm:ss.s{precision}" format, TIMESTAMP_WITH_LOCAL_TIMEZONE in + * "yyyy-MM-ddTHH:mm:ss.s{precision}'Z'" and output in the same format. + */ + ISO_8601 + } + + private TextFormatOptions() { + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatOptionsUtil.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatOptionsUtil.java new file mode 100644 index 00000000000..7ca4a5be058 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/base/TextFormatOptionsUtil.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.base; + +import org.apache.inlong.sort.formats.base.TextFormatOptions.MapNullKeyMode; +import org.apache.inlong.sort.formats.base.TextFormatOptions.TimestampFormat; + +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.api.ValidationException; + +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.inlong.sort.formats.base.TableFormatOptions.IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TextFormatOptions.CHARSET; +import static org.apache.inlong.sort.formats.base.TextFormatOptions.FAIL_ON_MISSING_FIELD; +import static org.apache.inlong.sort.formats.base.TextFormatOptions.MAP_NULL_KEY_MODE; +import static org.apache.inlong.sort.formats.base.TextFormatOptions.TIMESTAMP_FORMAT; + +/** Utils for {@link TextFormatOptions}. */ +public class TextFormatOptionsUtil { + + // -------------------------------------------------------------------------------------------- + // Option enumerations + // -------------------------------------------------------------------------------------------- + + public static final String SQL = "SQL"; + public static final String ISO_8601 = "ISO-8601"; + + public static final Set TIMESTAMP_FORMAT_ENUM = new HashSet<>(Arrays.asList(SQL, ISO_8601)); + + // The handling mode of null key for map data + public static final String MAP_NULL_KEY_MODE_FAIL = "FAIL"; + public static final String MAP_NULL_KEY_MODE_DROP = "DROP"; + public static final String MAP_NULL_KEY_MODE_LITERAL = "LITERAL"; + + // -------------------------------------------------------------------------------------------- + // Utilities + // -------------------------------------------------------------------------------------------- + + public static TimestampFormat getTimestampFormat(String timestampFormat) { + switch (timestampFormat) { + case SQL: + return TimestampFormat.SQL; + case ISO_8601: + return TimestampFormat.ISO_8601; + default: + throw new TableException( + String.format( + "Unsupported timestamp format '%s'. Validator should have checked that.", + timestampFormat)); + } + } + + /** + * Creates handling mode for null key map data. + * + *

See {@link #MAP_NULL_KEY_MODE_FAIL}, {@link #MAP_NULL_KEY_MODE_DROP}, and {@link + * #MAP_NULL_KEY_MODE_LITERAL} for more information. + */ + public static MapNullKeyMode getMapNullKeyMode(String mapNullKeyMode) { + switch (mapNullKeyMode.toUpperCase()) { + case MAP_NULL_KEY_MODE_FAIL: + return MapNullKeyMode.FAIL; + case MAP_NULL_KEY_MODE_DROP: + return MapNullKeyMode.DROP; + case MAP_NULL_KEY_MODE_LITERAL: + return MapNullKeyMode.LITERAL; + default: + throw new TableException( + String.format( + "Unsupported map null key handling mode '%s'. Validator should have checked that.", + mapNullKeyMode)); + } + } + + // -------------------------------------------------------------------------------------------- + // Validation + // -------------------------------------------------------------------------------------------- + + /** Validator for text decoding format. */ + public static void validateDecodingFormatOptions(ReadableConfig tableOptions) { + boolean failOnMissingField = tableOptions.get(FAIL_ON_MISSING_FIELD); + boolean ignoreParseErrors = tableOptions.get(IGNORE_ERRORS); + if (ignoreParseErrors && failOnMissingField) { + throw new ValidationException( + String.format("%s and %s shouldn't both be true.", + FAIL_ON_MISSING_FIELD.key(), + IGNORE_ERRORS.key())); + } + validateTimestampFormat(tableOptions); + validateCharset(tableOptions); + } + + /** Validator for text encoding format. */ + public static void validateEncodingFormatOptions(ReadableConfig tableOptions) { + Set nullKeyModes = + Arrays.stream(MapNullKeyMode.values()) + .map(Objects::toString) + .collect(Collectors.toSet()); + if (!nullKeyModes.contains(tableOptions.get(MAP_NULL_KEY_MODE).toUpperCase())) { + throw new ValidationException( + String.format( + "Unsupported value '%s' for option %s. Supported values are %s.", + tableOptions.get(MAP_NULL_KEY_MODE), + MAP_NULL_KEY_MODE.key(), + nullKeyModes)); + } + validateTimestampFormat(tableOptions); + validateCharset(tableOptions); + } + + /** Validates timestamp format which value should be SQL or ISO-8601. */ + static void validateTimestampFormat(ReadableConfig tableOptions) { + String timestampFormat = tableOptions.get(TIMESTAMP_FORMAT); + if (!TIMESTAMP_FORMAT_ENUM.contains(timestampFormat)) { + throw new ValidationException( + String.format( + "Unsupported value '%s' for %s. Supported values are [SQL, ISO-8601].", + timestampFormat, TIMESTAMP_FORMAT.key())); + } + } + + public static void validateCharset(ReadableConfig tableOptions) { + String charset = tableOptions.get(CHARSET); + try { + Charset.forName(charset); + } catch (Exception e) { + throw new ValidationException(String.format("Charset %s is not supported.", charset)); + } + } + + private TextFormatOptionsUtil() { + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/binlog/InLongBinlog.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/binlog/InLongBinlog.java new file mode 100644 index 00000000000..487fc6483e4 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/binlog/InLongBinlog.java @@ -0,0 +1,4622 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: InLongBinlog.proto + +package org.apache.inlong.sort.formats.binlog; + +public final class InLongBinlog { + private InLongBinlog() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + /** + * Protobuf enum {@code org.apache.inlong.sort.formats.binlog.EventType} + */ + public enum EventType + implements com.google.protobuf.ProtocolMessageEnum { + /** + * INSERT = 1; + */ + INSERT(0, 1), + /** + * UPDATE = 2; + */ + UPDATE(1, 2), + /** + * DELETE = 3; + */ + DELETE(2, 3), + /** + * CREATE = 4; + */ + CREATE(3, 4), + /** + * ALTER = 5; + */ + ALTER(4, 5), + /** + * ERASE = 6; + */ + ERASE(5, 6), + /** + * QUERY = 7; + */ + QUERY(6, 7), + /** + * GTID = 8; + */ + GTID(7, 8), + ; + + /** + * INSERT = 1; + */ + public static final int INSERT_VALUE = 1; + /** + * UPDATE = 2; + */ + public static final int UPDATE_VALUE = 2; + /** + * DELETE = 3; + */ + public static final int DELETE_VALUE = 3; + /** + * CREATE = 4; + */ + public static final int CREATE_VALUE = 4; + /** + * ALTER = 5; + */ + public static final int ALTER_VALUE = 5; + /** + * ERASE = 6; + */ + public static final int ERASE_VALUE = 6; + /** + * QUERY = 7; + */ + public static final int QUERY_VALUE = 7; + /** + * GTID = 8; + */ + public static final int GTID_VALUE = 8; + + + public final int getNumber() { return value; } + + public static EventType valueOf(int value) { + switch (value) { + case 1: return INSERT; + case 2: return UPDATE; + case 3: return DELETE; + case 4: return CREATE; + case 5: return ALTER; + case 6: return ERASE; + case 7: return QUERY; + case 8: return GTID; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public EventType findValueByNumber(int number) { + return EventType.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.getDescriptor().getEnumTypes().get(0); + } + + private static final EventType[] VALUES = values(); + + public static EventType valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private EventType(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:org.apache.inlong.sort.formats.binlog.EventType) + } + + public interface ColumnOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional int32 index = 1; + /** + * optional int32 index = 1; + */ + boolean hasIndex(); + /** + * optional int32 index = 1; + */ + int getIndex(); + + // optional int32 sqlType = 2; + /** + * optional int32 sqlType = 2; + */ + boolean hasSqlType(); + /** + * optional int32 sqlType = 2; + */ + int getSqlType(); + + // optional string name = 3; + /** + * optional string name = 3; + */ + boolean hasName(); + /** + * optional string name = 3; + */ + java.lang.String getName(); + /** + * optional string name = 3; + */ + com.google.protobuf.ByteString + getNameBytes(); + + // optional bool isKey = 4; + /** + * optional bool isKey = 4; + */ + boolean hasIsKey(); + /** + * optional bool isKey = 4; + */ + boolean getIsKey(); + + // optional bool updated = 5; + /** + * optional bool updated = 5; + */ + boolean hasUpdated(); + /** + * optional bool updated = 5; + */ + boolean getUpdated(); + + // optional bool isNull = 6 [default = false]; + /** + * optional bool isNull = 6 [default = false]; + */ + boolean hasIsNull(); + /** + * optional bool isNull = 6 [default = false]; + */ + boolean getIsNull(); + + // repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + java.util.List + getPropsList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair getProps(int index); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + int getPropsCount(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + java.util.List + getPropsOrBuilderList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder getPropsOrBuilder( + int index); + + // optional string value = 8; + /** + * optional string value = 8; + */ + boolean hasValue(); + /** + * optional string value = 8; + */ + java.lang.String getValue(); + /** + * optional string value = 8; + */ + com.google.protobuf.ByteString + getValueBytes(); + + // optional int32 length = 9; + /** + * optional int32 length = 9; + */ + boolean hasLength(); + /** + * optional int32 length = 9; + */ + int getLength(); + } + /** + * Protobuf type {@code org.apache.inlong.sort.formats.binlog.Column} + */ + public static final class Column extends + com.google.protobuf.GeneratedMessage + implements ColumnOrBuilder { + // Use Column.newBuilder() to construct. + private Column(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private Column(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final Column defaultInstance; + public static Column getDefaultInstance() { + return defaultInstance; + } + + public Column getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private Column( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + index_ = input.readInt32(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + sqlType_ = input.readInt32(); + break; + } + case 26: { + bitField0_ |= 0x00000004; + name_ = input.readBytes(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + isKey_ = input.readBool(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + updated_ = input.readBool(); + break; + } + case 48: { + bitField0_ |= 0x00000020; + isNull_ = input.readBool(); + break; + } + case 58: { + if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + props_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000040; + } + props_.add(input.readMessage(org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.PARSER, extensionRegistry)); + break; + } + case 66: { + bitField0_ |= 0x00000040; + value_ = input.readBytes(); + break; + } + case 72: { + bitField0_ |= 0x00000080; + length_ = input.readInt32(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + props_ = java.util.Collections.unmodifiableList(props_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Column_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Column_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.class, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public Column parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Column(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // optional int32 index = 1; + public static final int INDEX_FIELD_NUMBER = 1; + private int index_; + /** + * optional int32 index = 1; + */ + public boolean hasIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * optional int32 index = 1; + */ + public int getIndex() { + return index_; + } + + // optional int32 sqlType = 2; + public static final int SQLTYPE_FIELD_NUMBER = 2; + private int sqlType_; + /** + * optional int32 sqlType = 2; + */ + public boolean hasSqlType() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional int32 sqlType = 2; + */ + public int getSqlType() { + return sqlType_; + } + + // optional string name = 3; + public static final int NAME_FIELD_NUMBER = 3; + private java.lang.Object name_; + /** + * optional string name = 3; + */ + public boolean hasName() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional string name = 3; + */ + public java.lang.String getName() { + java.lang.Object ref = name_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + name_ = s; + } + return s; + } + } + /** + * optional string name = 3; + */ + public com.google.protobuf.ByteString + getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional bool isKey = 4; + public static final int ISKEY_FIELD_NUMBER = 4; + private boolean isKey_; + /** + * optional bool isKey = 4; + */ + public boolean hasIsKey() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional bool isKey = 4; + */ + public boolean getIsKey() { + return isKey_; + } + + // optional bool updated = 5; + public static final int UPDATED_FIELD_NUMBER = 5; + private boolean updated_; + /** + * optional bool updated = 5; + */ + public boolean hasUpdated() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional bool updated = 5; + */ + public boolean getUpdated() { + return updated_; + } + + // optional bool isNull = 6 [default = false]; + public static final int ISNULL_FIELD_NUMBER = 6; + private boolean isNull_; + /** + * optional bool isNull = 6 [default = false]; + */ + public boolean hasIsNull() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bool isNull = 6 [default = false]; + */ + public boolean getIsNull() { + return isNull_; + } + + // repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + public static final int PROPS_FIELD_NUMBER = 7; + private java.util.List props_; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public java.util.List getPropsList() { + return props_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public java.util.List + getPropsOrBuilderList() { + return props_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public int getPropsCount() { + return props_.size(); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair getProps(int index) { + return props_.get(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder getPropsOrBuilder( + int index) { + return props_.get(index); + } + + // optional string value = 8; + public static final int VALUE_FIELD_NUMBER = 8; + private java.lang.Object value_; + /** + * optional string value = 8; + */ + public boolean hasValue() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional string value = 8; + */ + public java.lang.String getValue() { + java.lang.Object ref = value_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + value_ = s; + } + return s; + } + } + /** + * optional string value = 8; + */ + public com.google.protobuf.ByteString + getValueBytes() { + java.lang.Object ref = value_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + value_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional int32 length = 9; + public static final int LENGTH_FIELD_NUMBER = 9; + private int length_; + /** + * optional int32 length = 9; + */ + public boolean hasLength() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional int32 length = 9; + */ + public int getLength() { + return length_; + } + + private void initFields() { + index_ = 0; + sqlType_ = 0; + name_ = ""; + isKey_ = false; + updated_ = false; + isNull_ = false; + props_ = java.util.Collections.emptyList(); + value_ = ""; + length_ = 0; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt32(1, index_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt32(2, sqlType_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBytes(3, getNameBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBool(4, isKey_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBool(5, updated_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeBool(6, isNull_); + } + for (int i = 0; i < props_.size(); i++) { + output.writeMessage(7, props_.get(i)); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeBytes(8, getValueBytes()); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + output.writeInt32(9, length_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(1, index_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(2, sqlType_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(3, getNameBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(4, isKey_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(5, updated_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(6, isNull_); + } + for (int i = 0; i < props_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(7, props_.get(i)); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(8, getValueBytes()); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(9, length_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.inlong.sort.formats.binlog.InLongBinlog.Column prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.apache.inlong.sort.formats.binlog.Column} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Column_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Column_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.class, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder.class); + } + + // Construct using org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getPropsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + index_ = 0; + bitField0_ = (bitField0_ & ~0x00000001); + sqlType_ = 0; + bitField0_ = (bitField0_ & ~0x00000002); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + isKey_ = false; + bitField0_ = (bitField0_ & ~0x00000008); + updated_ = false; + bitField0_ = (bitField0_ & ~0x00000010); + isNull_ = false; + bitField0_ = (bitField0_ & ~0x00000020); + if (propsBuilder_ == null) { + props_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); + } else { + propsBuilder_.clear(); + } + value_ = ""; + bitField0_ = (bitField0_ & ~0x00000080); + length_ = 0; + bitField0_ = (bitField0_ & ~0x00000100); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Column_descriptor; + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column getDefaultInstanceForType() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.getDefaultInstance(); + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column build() { + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column buildPartial() { + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column result = new org.apache.inlong.sort.formats.binlog.InLongBinlog.Column(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.index_ = index_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.sqlType_ = sqlType_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.isKey_ = isKey_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.updated_ = updated_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.isNull_ = isNull_; + if (propsBuilder_ == null) { + if (((bitField0_ & 0x00000040) == 0x00000040)) { + props_ = java.util.Collections.unmodifiableList(props_); + bitField0_ = (bitField0_ & ~0x00000040); + } + result.props_ = props_; + } else { + result.props_ = propsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000040; + } + result.value_ = value_; + if (((from_bitField0_ & 0x00000100) == 0x00000100)) { + to_bitField0_ |= 0x00000080; + } + result.length_ = length_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.inlong.sort.formats.binlog.InLongBinlog.Column) { + return mergeFrom((org.apache.inlong.sort.formats.binlog.InLongBinlog.Column)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.inlong.sort.formats.binlog.InLongBinlog.Column other) { + if (other == org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.getDefaultInstance()) return this; + if (other.hasIndex()) { + setIndex(other.getIndex()); + } + if (other.hasSqlType()) { + setSqlType(other.getSqlType()); + } + if (other.hasName()) { + bitField0_ |= 0x00000004; + name_ = other.name_; + onChanged(); + } + if (other.hasIsKey()) { + setIsKey(other.getIsKey()); + } + if (other.hasUpdated()) { + setUpdated(other.getUpdated()); + } + if (other.hasIsNull()) { + setIsNull(other.getIsNull()); + } + if (propsBuilder_ == null) { + if (!other.props_.isEmpty()) { + if (props_.isEmpty()) { + props_ = other.props_; + bitField0_ = (bitField0_ & ~0x00000040); + } else { + ensurePropsIsMutable(); + props_.addAll(other.props_); + } + onChanged(); + } + } else { + if (!other.props_.isEmpty()) { + if (propsBuilder_.isEmpty()) { + propsBuilder_.dispose(); + propsBuilder_ = null; + props_ = other.props_; + bitField0_ = (bitField0_ & ~0x00000040); + propsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getPropsFieldBuilder() : null; + } else { + propsBuilder_.addAllMessages(other.props_); + } + } + } + if (other.hasValue()) { + bitField0_ |= 0x00000080; + value_ = other.value_; + onChanged(); + } + if (other.hasLength()) { + setLength(other.getLength()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.inlong.sort.formats.binlog.InLongBinlog.Column) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // optional int32 index = 1; + private int index_ ; + /** + * optional int32 index = 1; + */ + public boolean hasIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * optional int32 index = 1; + */ + public int getIndex() { + return index_; + } + /** + * optional int32 index = 1; + */ + public Builder setIndex(int value) { + bitField0_ |= 0x00000001; + index_ = value; + onChanged(); + return this; + } + /** + * optional int32 index = 1; + */ + public Builder clearIndex() { + bitField0_ = (bitField0_ & ~0x00000001); + index_ = 0; + onChanged(); + return this; + } + + // optional int32 sqlType = 2; + private int sqlType_ ; + /** + * optional int32 sqlType = 2; + */ + public boolean hasSqlType() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional int32 sqlType = 2; + */ + public int getSqlType() { + return sqlType_; + } + /** + * optional int32 sqlType = 2; + */ + public Builder setSqlType(int value) { + bitField0_ |= 0x00000002; + sqlType_ = value; + onChanged(); + return this; + } + /** + * optional int32 sqlType = 2; + */ + public Builder clearSqlType() { + bitField0_ = (bitField0_ & ~0x00000002); + sqlType_ = 0; + onChanged(); + return this; + } + + // optional string name = 3; + private java.lang.Object name_ = ""; + /** + * optional string name = 3; + */ + public boolean hasName() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional string name = 3; + */ + public java.lang.String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + name_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string name = 3; + */ + public com.google.protobuf.ByteString + getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string name = 3; + */ + public Builder setName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + name_ = value; + onChanged(); + return this; + } + /** + * optional string name = 3; + */ + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000004); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + /** + * optional string name = 3; + */ + public Builder setNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + name_ = value; + onChanged(); + return this; + } + + // optional bool isKey = 4; + private boolean isKey_ ; + /** + * optional bool isKey = 4; + */ + public boolean hasIsKey() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional bool isKey = 4; + */ + public boolean getIsKey() { + return isKey_; + } + /** + * optional bool isKey = 4; + */ + public Builder setIsKey(boolean value) { + bitField0_ |= 0x00000008; + isKey_ = value; + onChanged(); + return this; + } + /** + * optional bool isKey = 4; + */ + public Builder clearIsKey() { + bitField0_ = (bitField0_ & ~0x00000008); + isKey_ = false; + onChanged(); + return this; + } + + // optional bool updated = 5; + private boolean updated_ ; + /** + * optional bool updated = 5; + */ + public boolean hasUpdated() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional bool updated = 5; + */ + public boolean getUpdated() { + return updated_; + } + /** + * optional bool updated = 5; + */ + public Builder setUpdated(boolean value) { + bitField0_ |= 0x00000010; + updated_ = value; + onChanged(); + return this; + } + /** + * optional bool updated = 5; + */ + public Builder clearUpdated() { + bitField0_ = (bitField0_ & ~0x00000010); + updated_ = false; + onChanged(); + return this; + } + + // optional bool isNull = 6 [default = false]; + private boolean isNull_ ; + /** + * optional bool isNull = 6 [default = false]; + */ + public boolean hasIsNull() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bool isNull = 6 [default = false]; + */ + public boolean getIsNull() { + return isNull_; + } + /** + * optional bool isNull = 6 [default = false]; + */ + public Builder setIsNull(boolean value) { + bitField0_ |= 0x00000020; + isNull_ = value; + onChanged(); + return this; + } + /** + * optional bool isNull = 6 [default = false]; + */ + public Builder clearIsNull() { + bitField0_ = (bitField0_ & ~0x00000020); + isNull_ = false; + onChanged(); + return this; + } + + // repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + private java.util.List props_ = + java.util.Collections.emptyList(); + private void ensurePropsIsMutable() { + if (!((bitField0_ & 0x00000040) == 0x00000040)) { + props_ = new java.util.ArrayList(props_); + bitField0_ |= 0x00000040; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder> propsBuilder_; + + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public java.util.List getPropsList() { + if (propsBuilder_ == null) { + return java.util.Collections.unmodifiableList(props_); + } else { + return propsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public int getPropsCount() { + if (propsBuilder_ == null) { + return props_.size(); + } else { + return propsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair getProps(int index) { + if (propsBuilder_ == null) { + return props_.get(index); + } else { + return propsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder setProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair value) { + if (propsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePropsIsMutable(); + props_.set(index, value); + onChanged(); + } else { + propsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder setProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder builderForValue) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.set(index, builderForValue.build()); + onChanged(); + } else { + propsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder addProps(org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair value) { + if (propsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePropsIsMutable(); + props_.add(value); + onChanged(); + } else { + propsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder addProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair value) { + if (propsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePropsIsMutable(); + props_.add(index, value); + onChanged(); + } else { + propsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder addProps( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder builderForValue) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.add(builderForValue.build()); + onChanged(); + } else { + propsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder addProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder builderForValue) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.add(index, builderForValue.build()); + onChanged(); + } else { + propsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder addAllProps( + java.lang.Iterable values) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + super.addAll(values, props_); + onChanged(); + } else { + propsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder clearProps() { + if (propsBuilder_ == null) { + props_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); + onChanged(); + } else { + propsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public Builder removeProps(int index) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.remove(index); + onChanged(); + } else { + propsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder getPropsBuilder( + int index) { + return getPropsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder getPropsOrBuilder( + int index) { + if (propsBuilder_ == null) { + return props_.get(index); } else { + return propsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public java.util.List + getPropsOrBuilderList() { + if (propsBuilder_ != null) { + return propsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(props_); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder addPropsBuilder() { + return getPropsFieldBuilder().addBuilder( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder addPropsBuilder( + int index) { + return getPropsFieldBuilder().addBuilder( + index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 7; + */ + public java.util.List + getPropsBuilderList() { + return getPropsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder> + getPropsFieldBuilder() { + if (propsBuilder_ == null) { + propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder>( + props_, + ((bitField0_ & 0x00000040) == 0x00000040), + getParentForChildren(), + isClean()); + props_ = null; + } + return propsBuilder_; + } + + // optional string value = 8; + private java.lang.Object value_ = ""; + /** + * optional string value = 8; + */ + public boolean hasValue() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional string value = 8; + */ + public java.lang.String getValue() { + java.lang.Object ref = value_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + value_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string value = 8; + */ + public com.google.protobuf.ByteString + getValueBytes() { + java.lang.Object ref = value_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + value_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string value = 8; + */ + public Builder setValue( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000080; + value_ = value; + onChanged(); + return this; + } + /** + * optional string value = 8; + */ + public Builder clearValue() { + bitField0_ = (bitField0_ & ~0x00000080); + value_ = getDefaultInstance().getValue(); + onChanged(); + return this; + } + /** + * optional string value = 8; + */ + public Builder setValueBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000080; + value_ = value; + onChanged(); + return this; + } + + // optional int32 length = 9; + private int length_ ; + /** + * optional int32 length = 9; + */ + public boolean hasLength() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + /** + * optional int32 length = 9; + */ + public int getLength() { + return length_; + } + /** + * optional int32 length = 9; + */ + public Builder setLength(int value) { + bitField0_ |= 0x00000100; + length_ = value; + onChanged(); + return this; + } + /** + * optional int32 length = 9; + */ + public Builder clearLength() { + bitField0_ = (bitField0_ & ~0x00000100); + length_ = 0; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.inlong.sort.formats.binlog.Column) + } + + static { + defaultInstance = new Column(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.inlong.sort.formats.binlog.Column) + } + + public interface RowDataOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + java.util.List + getBeforeColumnsList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column getBeforeColumns(int index); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + int getBeforeColumnsCount(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + java.util.List + getBeforeColumnsOrBuilderList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder getBeforeColumnsOrBuilder( + int index); + + // repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + java.util.List + getAfterColumnsList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column getAfterColumns(int index); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + int getAfterColumnsCount(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + java.util.List + getAfterColumnsOrBuilderList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder getAfterColumnsOrBuilder( + int index); + + // repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + java.util.List + getPropsList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair getProps(int index); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + int getPropsCount(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + java.util.List + getPropsOrBuilderList(); + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder getPropsOrBuilder( + int index); + + // optional string instanceName = 4; + /** + * optional string instanceName = 4; + */ + boolean hasInstanceName(); + /** + * optional string instanceName = 4; + */ + java.lang.String getInstanceName(); + /** + * optional string instanceName = 4; + */ + com.google.protobuf.ByteString + getInstanceNameBytes(); + + // optional string schemaName = 5; + /** + * optional string schemaName = 5; + */ + boolean hasSchemaName(); + /** + * optional string schemaName = 5; + */ + java.lang.String getSchemaName(); + /** + * optional string schemaName = 5; + */ + com.google.protobuf.ByteString + getSchemaNameBytes(); + + // optional string tableName = 6; + /** + * optional string tableName = 6; + */ + boolean hasTableName(); + /** + * optional string tableName = 6; + */ + java.lang.String getTableName(); + /** + * optional string tableName = 6; + */ + com.google.protobuf.ByteString + getTableNameBytes(); + + // optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + boolean hasEventType(); + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType getEventType(); + + // optional int64 executeTime = 8; + /** + * optional int64 executeTime = 8; + */ + boolean hasExecuteTime(); + /** + * optional int64 executeTime = 8; + */ + long getExecuteTime(); + + // optional int64 executeOrder = 9; + /** + * optional int64 executeOrder = 9; + */ + boolean hasExecuteOrder(); + /** + * optional int64 executeOrder = 9; + */ + long getExecuteOrder(); + + // optional string transferIp = 10; + /** + * optional string transferIp = 10; + */ + boolean hasTransferIp(); + /** + * optional string transferIp = 10; + */ + java.lang.String getTransferIp(); + /** + * optional string transferIp = 10; + */ + com.google.protobuf.ByteString + getTransferIpBytes(); + } + /** + * Protobuf type {@code org.apache.inlong.sort.formats.binlog.RowData} + */ + public static final class RowData extends + com.google.protobuf.GeneratedMessage + implements RowDataOrBuilder { + // Use RowData.newBuilder() to construct. + private RowData(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private RowData(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final RowData defaultInstance; + public static RowData getDefaultInstance() { + return defaultInstance; + } + + public RowData getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private RowData( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + beforeColumns_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + beforeColumns_.add(input.readMessage(org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.PARSER, extensionRegistry)); + break; + } + case 18: { + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + afterColumns_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000002; + } + afterColumns_.add(input.readMessage(org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.PARSER, extensionRegistry)); + break; + } + case 26: { + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + props_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000004; + } + props_.add(input.readMessage(org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.PARSER, extensionRegistry)); + break; + } + case 34: { + bitField0_ |= 0x00000001; + instanceName_ = input.readBytes(); + break; + } + case 42: { + bitField0_ |= 0x00000002; + schemaName_ = input.readBytes(); + break; + } + case 50: { + bitField0_ |= 0x00000004; + tableName_ = input.readBytes(); + break; + } + case 56: { + int rawValue = input.readEnum(); + org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType value = org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(7, rawValue); + } else { + bitField0_ |= 0x00000008; + eventType_ = value; + } + break; + } + case 64: { + bitField0_ |= 0x00000010; + executeTime_ = input.readInt64(); + break; + } + case 72: { + bitField0_ |= 0x00000020; + executeOrder_ = input.readInt64(); + break; + } + case 82: { + bitField0_ |= 0x00000040; + transferIp_ = input.readBytes(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + beforeColumns_ = java.util.Collections.unmodifiableList(beforeColumns_); + } + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + afterColumns_ = java.util.Collections.unmodifiableList(afterColumns_); + } + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + props_ = java.util.Collections.unmodifiableList(props_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_RowData_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_RowData_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData.class, org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public RowData parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RowData(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + public static final int BEFORECOLUMNS_FIELD_NUMBER = 1; + private java.util.List beforeColumns_; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public java.util.List getBeforeColumnsList() { + return beforeColumns_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public java.util.List + getBeforeColumnsOrBuilderList() { + return beforeColumns_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public int getBeforeColumnsCount() { + return beforeColumns_.size(); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column getBeforeColumns(int index) { + return beforeColumns_.get(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder getBeforeColumnsOrBuilder( + int index) { + return beforeColumns_.get(index); + } + + // repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + public static final int AFTERCOLUMNS_FIELD_NUMBER = 2; + private java.util.List afterColumns_; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public java.util.List getAfterColumnsList() { + return afterColumns_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public java.util.List + getAfterColumnsOrBuilderList() { + return afterColumns_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public int getAfterColumnsCount() { + return afterColumns_.size(); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column getAfterColumns(int index) { + return afterColumns_.get(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder getAfterColumnsOrBuilder( + int index) { + return afterColumns_.get(index); + } + + // repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + public static final int PROPS_FIELD_NUMBER = 3; + private java.util.List props_; + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public java.util.List getPropsList() { + return props_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public java.util.List + getPropsOrBuilderList() { + return props_; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public int getPropsCount() { + return props_.size(); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair getProps(int index) { + return props_.get(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder getPropsOrBuilder( + int index) { + return props_.get(index); + } + + // optional string instanceName = 4; + public static final int INSTANCENAME_FIELD_NUMBER = 4; + private java.lang.Object instanceName_; + /** + * optional string instanceName = 4; + */ + public boolean hasInstanceName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * optional string instanceName = 4; + */ + public java.lang.String getInstanceName() { + java.lang.Object ref = instanceName_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + instanceName_ = s; + } + return s; + } + } + /** + * optional string instanceName = 4; + */ + public com.google.protobuf.ByteString + getInstanceNameBytes() { + java.lang.Object ref = instanceName_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + instanceName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string schemaName = 5; + public static final int SCHEMANAME_FIELD_NUMBER = 5; + private java.lang.Object schemaName_; + /** + * optional string schemaName = 5; + */ + public boolean hasSchemaName() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional string schemaName = 5; + */ + public java.lang.String getSchemaName() { + java.lang.Object ref = schemaName_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + schemaName_ = s; + } + return s; + } + } + /** + * optional string schemaName = 5; + */ + public com.google.protobuf.ByteString + getSchemaNameBytes() { + java.lang.Object ref = schemaName_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + schemaName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string tableName = 6; + public static final int TABLENAME_FIELD_NUMBER = 6; + private java.lang.Object tableName_; + /** + * optional string tableName = 6; + */ + public boolean hasTableName() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional string tableName = 6; + */ + public java.lang.String getTableName() { + java.lang.Object ref = tableName_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + tableName_ = s; + } + return s; + } + } + /** + * optional string tableName = 6; + */ + public com.google.protobuf.ByteString + getTableNameBytes() { + java.lang.Object ref = tableName_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + tableName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + public static final int EVENTTYPE_FIELD_NUMBER = 7; + private org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType eventType_; + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + public boolean hasEventType() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType getEventType() { + return eventType_; + } + + // optional int64 executeTime = 8; + public static final int EXECUTETIME_FIELD_NUMBER = 8; + private long executeTime_; + /** + * optional int64 executeTime = 8; + */ + public boolean hasExecuteTime() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional int64 executeTime = 8; + */ + public long getExecuteTime() { + return executeTime_; + } + + // optional int64 executeOrder = 9; + public static final int EXECUTEORDER_FIELD_NUMBER = 9; + private long executeOrder_; + /** + * optional int64 executeOrder = 9; + */ + public boolean hasExecuteOrder() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional int64 executeOrder = 9; + */ + public long getExecuteOrder() { + return executeOrder_; + } + + // optional string transferIp = 10; + public static final int TRANSFERIP_FIELD_NUMBER = 10; + private java.lang.Object transferIp_; + /** + * optional string transferIp = 10; + */ + public boolean hasTransferIp() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional string transferIp = 10; + */ + public java.lang.String getTransferIp() { + java.lang.Object ref = transferIp_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + transferIp_ = s; + } + return s; + } + } + /** + * optional string transferIp = 10; + */ + public com.google.protobuf.ByteString + getTransferIpBytes() { + java.lang.Object ref = transferIp_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + transferIp_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + beforeColumns_ = java.util.Collections.emptyList(); + afterColumns_ = java.util.Collections.emptyList(); + props_ = java.util.Collections.emptyList(); + instanceName_ = ""; + schemaName_ = ""; + tableName_ = ""; + eventType_ = org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType.UPDATE; + executeTime_ = 0L; + executeOrder_ = 0L; + transferIp_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < beforeColumns_.size(); i++) { + output.writeMessage(1, beforeColumns_.get(i)); + } + for (int i = 0; i < afterColumns_.size(); i++) { + output.writeMessage(2, afterColumns_.get(i)); + } + for (int i = 0; i < props_.size(); i++) { + output.writeMessage(3, props_.get(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(4, getInstanceNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(5, getSchemaNameBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBytes(6, getTableNameBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeEnum(7, eventType_.getNumber()); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt64(8, executeTime_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeInt64(9, executeOrder_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeBytes(10, getTransferIpBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < beforeColumns_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, beforeColumns_.get(i)); + } + for (int i = 0; i < afterColumns_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, afterColumns_.get(i)); + } + for (int i = 0; i < props_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, props_.get(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(4, getInstanceNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(5, getSchemaNameBytes()); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(6, getTableNameBytes()); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(7, eventType_.getNumber()); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(8, executeTime_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(9, executeOrder_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(10, getTransferIpBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.apache.inlong.sort.formats.binlog.RowData} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.inlong.sort.formats.binlog.InLongBinlog.RowDataOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_RowData_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_RowData_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData.class, org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData.Builder.class); + } + + // Construct using org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getBeforeColumnsFieldBuilder(); + getAfterColumnsFieldBuilder(); + getPropsFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (beforeColumnsBuilder_ == null) { + beforeColumns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + beforeColumnsBuilder_.clear(); + } + if (afterColumnsBuilder_ == null) { + afterColumns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + afterColumnsBuilder_.clear(); + } + if (propsBuilder_ == null) { + props_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + } else { + propsBuilder_.clear(); + } + instanceName_ = ""; + bitField0_ = (bitField0_ & ~0x00000008); + schemaName_ = ""; + bitField0_ = (bitField0_ & ~0x00000010); + tableName_ = ""; + bitField0_ = (bitField0_ & ~0x00000020); + eventType_ = org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType.UPDATE; + bitField0_ = (bitField0_ & ~0x00000040); + executeTime_ = 0L; + bitField0_ = (bitField0_ & ~0x00000080); + executeOrder_ = 0L; + bitField0_ = (bitField0_ & ~0x00000100); + transferIp_ = ""; + bitField0_ = (bitField0_ & ~0x00000200); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_RowData_descriptor; + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData getDefaultInstanceForType() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData.getDefaultInstance(); + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData build() { + org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData buildPartial() { + org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData result = new org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (beforeColumnsBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + beforeColumns_ = java.util.Collections.unmodifiableList(beforeColumns_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.beforeColumns_ = beforeColumns_; + } else { + result.beforeColumns_ = beforeColumnsBuilder_.build(); + } + if (afterColumnsBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002)) { + afterColumns_ = java.util.Collections.unmodifiableList(afterColumns_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.afterColumns_ = afterColumns_; + } else { + result.afterColumns_ = afterColumnsBuilder_.build(); + } + if (propsBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004)) { + props_ = java.util.Collections.unmodifiableList(props_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.props_ = props_; + } else { + result.props_ = propsBuilder_.build(); + } + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000001; + } + result.instanceName_ = instanceName_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000002; + } + result.schemaName_ = schemaName_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000004; + } + result.tableName_ = tableName_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000008; + } + result.eventType_ = eventType_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000010; + } + result.executeTime_ = executeTime_; + if (((from_bitField0_ & 0x00000100) == 0x00000100)) { + to_bitField0_ |= 0x00000020; + } + result.executeOrder_ = executeOrder_; + if (((from_bitField0_ & 0x00000200) == 0x00000200)) { + to_bitField0_ |= 0x00000040; + } + result.transferIp_ = transferIp_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData) { + return mergeFrom((org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData other) { + if (other == org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData.getDefaultInstance()) return this; + if (beforeColumnsBuilder_ == null) { + if (!other.beforeColumns_.isEmpty()) { + if (beforeColumns_.isEmpty()) { + beforeColumns_ = other.beforeColumns_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureBeforeColumnsIsMutable(); + beforeColumns_.addAll(other.beforeColumns_); + } + onChanged(); + } + } else { + if (!other.beforeColumns_.isEmpty()) { + if (beforeColumnsBuilder_.isEmpty()) { + beforeColumnsBuilder_.dispose(); + beforeColumnsBuilder_ = null; + beforeColumns_ = other.beforeColumns_; + bitField0_ = (bitField0_ & ~0x00000001); + beforeColumnsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getBeforeColumnsFieldBuilder() : null; + } else { + beforeColumnsBuilder_.addAllMessages(other.beforeColumns_); + } + } + } + if (afterColumnsBuilder_ == null) { + if (!other.afterColumns_.isEmpty()) { + if (afterColumns_.isEmpty()) { + afterColumns_ = other.afterColumns_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureAfterColumnsIsMutable(); + afterColumns_.addAll(other.afterColumns_); + } + onChanged(); + } + } else { + if (!other.afterColumns_.isEmpty()) { + if (afterColumnsBuilder_.isEmpty()) { + afterColumnsBuilder_.dispose(); + afterColumnsBuilder_ = null; + afterColumns_ = other.afterColumns_; + bitField0_ = (bitField0_ & ~0x00000002); + afterColumnsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getAfterColumnsFieldBuilder() : null; + } else { + afterColumnsBuilder_.addAllMessages(other.afterColumns_); + } + } + } + if (propsBuilder_ == null) { + if (!other.props_.isEmpty()) { + if (props_.isEmpty()) { + props_ = other.props_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensurePropsIsMutable(); + props_.addAll(other.props_); + } + onChanged(); + } + } else { + if (!other.props_.isEmpty()) { + if (propsBuilder_.isEmpty()) { + propsBuilder_.dispose(); + propsBuilder_ = null; + props_ = other.props_; + bitField0_ = (bitField0_ & ~0x00000004); + propsBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getPropsFieldBuilder() : null; + } else { + propsBuilder_.addAllMessages(other.props_); + } + } + } + if (other.hasInstanceName()) { + bitField0_ |= 0x00000008; + instanceName_ = other.instanceName_; + onChanged(); + } + if (other.hasSchemaName()) { + bitField0_ |= 0x00000010; + schemaName_ = other.schemaName_; + onChanged(); + } + if (other.hasTableName()) { + bitField0_ |= 0x00000020; + tableName_ = other.tableName_; + onChanged(); + } + if (other.hasEventType()) { + setEventType(other.getEventType()); + } + if (other.hasExecuteTime()) { + setExecuteTime(other.getExecuteTime()); + } + if (other.hasExecuteOrder()) { + setExecuteOrder(other.getExecuteOrder()); + } + if (other.hasTransferIp()) { + bitField0_ |= 0x00000200; + transferIp_ = other.transferIp_; + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.inlong.sort.formats.binlog.InLongBinlog.RowData) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + private java.util.List beforeColumns_ = + java.util.Collections.emptyList(); + private void ensureBeforeColumnsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + beforeColumns_ = new java.util.ArrayList(beforeColumns_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder> beforeColumnsBuilder_; + + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public java.util.List getBeforeColumnsList() { + if (beforeColumnsBuilder_ == null) { + return java.util.Collections.unmodifiableList(beforeColumns_); + } else { + return beforeColumnsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public int getBeforeColumnsCount() { + if (beforeColumnsBuilder_ == null) { + return beforeColumns_.size(); + } else { + return beforeColumnsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column getBeforeColumns(int index) { + if (beforeColumnsBuilder_ == null) { + return beforeColumns_.get(index); + } else { + return beforeColumnsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder setBeforeColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column value) { + if (beforeColumnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureBeforeColumnsIsMutable(); + beforeColumns_.set(index, value); + onChanged(); + } else { + beforeColumnsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder setBeforeColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder builderForValue) { + if (beforeColumnsBuilder_ == null) { + ensureBeforeColumnsIsMutable(); + beforeColumns_.set(index, builderForValue.build()); + onChanged(); + } else { + beforeColumnsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder addBeforeColumns(org.apache.inlong.sort.formats.binlog.InLongBinlog.Column value) { + if (beforeColumnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureBeforeColumnsIsMutable(); + beforeColumns_.add(value); + onChanged(); + } else { + beforeColumnsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder addBeforeColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column value) { + if (beforeColumnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureBeforeColumnsIsMutable(); + beforeColumns_.add(index, value); + onChanged(); + } else { + beforeColumnsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder addBeforeColumns( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder builderForValue) { + if (beforeColumnsBuilder_ == null) { + ensureBeforeColumnsIsMutable(); + beforeColumns_.add(builderForValue.build()); + onChanged(); + } else { + beforeColumnsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder addBeforeColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder builderForValue) { + if (beforeColumnsBuilder_ == null) { + ensureBeforeColumnsIsMutable(); + beforeColumns_.add(index, builderForValue.build()); + onChanged(); + } else { + beforeColumnsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder addAllBeforeColumns( + java.lang.Iterable values) { + if (beforeColumnsBuilder_ == null) { + ensureBeforeColumnsIsMutable(); + super.addAll(values, beforeColumns_); + onChanged(); + } else { + beforeColumnsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder clearBeforeColumns() { + if (beforeColumnsBuilder_ == null) { + beforeColumns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + beforeColumnsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public Builder removeBeforeColumns(int index) { + if (beforeColumnsBuilder_ == null) { + ensureBeforeColumnsIsMutable(); + beforeColumns_.remove(index); + onChanged(); + } else { + beforeColumnsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder getBeforeColumnsBuilder( + int index) { + return getBeforeColumnsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder getBeforeColumnsOrBuilder( + int index) { + if (beforeColumnsBuilder_ == null) { + return beforeColumns_.get(index); } else { + return beforeColumnsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public java.util.List + getBeforeColumnsOrBuilderList() { + if (beforeColumnsBuilder_ != null) { + return beforeColumnsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(beforeColumns_); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder addBeforeColumnsBuilder() { + return getBeforeColumnsFieldBuilder().addBuilder( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder addBeforeColumnsBuilder( + int index) { + return getBeforeColumnsFieldBuilder().addBuilder( + index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column beforeColumns = 1; + */ + public java.util.List + getBeforeColumnsBuilderList() { + return getBeforeColumnsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder> + getBeforeColumnsFieldBuilder() { + if (beforeColumnsBuilder_ == null) { + beforeColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder>( + beforeColumns_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + beforeColumns_ = null; + } + return beforeColumnsBuilder_; + } + + // repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + private java.util.List afterColumns_ = + java.util.Collections.emptyList(); + private void ensureAfterColumnsIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + afterColumns_ = new java.util.ArrayList(afterColumns_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder> afterColumnsBuilder_; + + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public java.util.List getAfterColumnsList() { + if (afterColumnsBuilder_ == null) { + return java.util.Collections.unmodifiableList(afterColumns_); + } else { + return afterColumnsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public int getAfterColumnsCount() { + if (afterColumnsBuilder_ == null) { + return afterColumns_.size(); + } else { + return afterColumnsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column getAfterColumns(int index) { + if (afterColumnsBuilder_ == null) { + return afterColumns_.get(index); + } else { + return afterColumnsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder setAfterColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column value) { + if (afterColumnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAfterColumnsIsMutable(); + afterColumns_.set(index, value); + onChanged(); + } else { + afterColumnsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder setAfterColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder builderForValue) { + if (afterColumnsBuilder_ == null) { + ensureAfterColumnsIsMutable(); + afterColumns_.set(index, builderForValue.build()); + onChanged(); + } else { + afterColumnsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder addAfterColumns(org.apache.inlong.sort.formats.binlog.InLongBinlog.Column value) { + if (afterColumnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAfterColumnsIsMutable(); + afterColumns_.add(value); + onChanged(); + } else { + afterColumnsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder addAfterColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column value) { + if (afterColumnsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureAfterColumnsIsMutable(); + afterColumns_.add(index, value); + onChanged(); + } else { + afterColumnsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder addAfterColumns( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder builderForValue) { + if (afterColumnsBuilder_ == null) { + ensureAfterColumnsIsMutable(); + afterColumns_.add(builderForValue.build()); + onChanged(); + } else { + afterColumnsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder addAfterColumns( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder builderForValue) { + if (afterColumnsBuilder_ == null) { + ensureAfterColumnsIsMutable(); + afterColumns_.add(index, builderForValue.build()); + onChanged(); + } else { + afterColumnsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder addAllAfterColumns( + java.lang.Iterable values) { + if (afterColumnsBuilder_ == null) { + ensureAfterColumnsIsMutable(); + super.addAll(values, afterColumns_); + onChanged(); + } else { + afterColumnsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder clearAfterColumns() { + if (afterColumnsBuilder_ == null) { + afterColumns_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + afterColumnsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public Builder removeAfterColumns(int index) { + if (afterColumnsBuilder_ == null) { + ensureAfterColumnsIsMutable(); + afterColumns_.remove(index); + onChanged(); + } else { + afterColumnsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder getAfterColumnsBuilder( + int index) { + return getAfterColumnsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder getAfterColumnsOrBuilder( + int index) { + if (afterColumnsBuilder_ == null) { + return afterColumns_.get(index); } else { + return afterColumnsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public java.util.List + getAfterColumnsOrBuilderList() { + if (afterColumnsBuilder_ != null) { + return afterColumnsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(afterColumns_); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder addAfterColumnsBuilder() { + return getAfterColumnsFieldBuilder().addBuilder( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder addAfterColumnsBuilder( + int index) { + return getAfterColumnsFieldBuilder().addBuilder( + index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Column afterColumns = 2; + */ + public java.util.List + getAfterColumnsBuilderList() { + return getAfterColumnsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder> + getAfterColumnsFieldBuilder() { + if (afterColumnsBuilder_ == null) { + afterColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Column, org.apache.inlong.sort.formats.binlog.InLongBinlog.Column.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.ColumnOrBuilder>( + afterColumns_, + ((bitField0_ & 0x00000002) == 0x00000002), + getParentForChildren(), + isClean()); + afterColumns_ = null; + } + return afterColumnsBuilder_; + } + + // repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + private java.util.List props_ = + java.util.Collections.emptyList(); + private void ensurePropsIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + props_ = new java.util.ArrayList(props_); + bitField0_ |= 0x00000004; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder> propsBuilder_; + + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public java.util.List getPropsList() { + if (propsBuilder_ == null) { + return java.util.Collections.unmodifiableList(props_); + } else { + return propsBuilder_.getMessageList(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public int getPropsCount() { + if (propsBuilder_ == null) { + return props_.size(); + } else { + return propsBuilder_.getCount(); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair getProps(int index) { + if (propsBuilder_ == null) { + return props_.get(index); + } else { + return propsBuilder_.getMessage(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder setProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair value) { + if (propsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePropsIsMutable(); + props_.set(index, value); + onChanged(); + } else { + propsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder setProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder builderForValue) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.set(index, builderForValue.build()); + onChanged(); + } else { + propsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder addProps(org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair value) { + if (propsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePropsIsMutable(); + props_.add(value); + onChanged(); + } else { + propsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder addProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair value) { + if (propsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePropsIsMutable(); + props_.add(index, value); + onChanged(); + } else { + propsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder addProps( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder builderForValue) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.add(builderForValue.build()); + onChanged(); + } else { + propsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder addProps( + int index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder builderForValue) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.add(index, builderForValue.build()); + onChanged(); + } else { + propsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder addAllProps( + java.lang.Iterable values) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + super.addAll(values, props_); + onChanged(); + } else { + propsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder clearProps() { + if (propsBuilder_ == null) { + props_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + } else { + propsBuilder_.clear(); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public Builder removeProps(int index) { + if (propsBuilder_ == null) { + ensurePropsIsMutable(); + props_.remove(index); + onChanged(); + } else { + propsBuilder_.remove(index); + } + return this; + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder getPropsBuilder( + int index) { + return getPropsFieldBuilder().getBuilder(index); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder getPropsOrBuilder( + int index) { + if (propsBuilder_ == null) { + return props_.get(index); } else { + return propsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public java.util.List + getPropsOrBuilderList() { + if (propsBuilder_ != null) { + return propsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(props_); + } + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder addPropsBuilder() { + return getPropsFieldBuilder().addBuilder( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder addPropsBuilder( + int index) { + return getPropsFieldBuilder().addBuilder( + index, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.getDefaultInstance()); + } + /** + * repeated .org.apache.inlong.sort.formats.binlog.Pair props = 3; + */ + public java.util.List + getPropsBuilderList() { + return getPropsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder> + getPropsFieldBuilder() { + if (propsBuilder_ == null) { + propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder, org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder>( + props_, + ((bitField0_ & 0x00000004) == 0x00000004), + getParentForChildren(), + isClean()); + props_ = null; + } + return propsBuilder_; + } + + // optional string instanceName = 4; + private java.lang.Object instanceName_ = ""; + /** + * optional string instanceName = 4; + */ + public boolean hasInstanceName() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional string instanceName = 4; + */ + public java.lang.String getInstanceName() { + java.lang.Object ref = instanceName_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + instanceName_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string instanceName = 4; + */ + public com.google.protobuf.ByteString + getInstanceNameBytes() { + java.lang.Object ref = instanceName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + instanceName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string instanceName = 4; + */ + public Builder setInstanceName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + instanceName_ = value; + onChanged(); + return this; + } + /** + * optional string instanceName = 4; + */ + public Builder clearInstanceName() { + bitField0_ = (bitField0_ & ~0x00000008); + instanceName_ = getDefaultInstance().getInstanceName(); + onChanged(); + return this; + } + /** + * optional string instanceName = 4; + */ + public Builder setInstanceNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + instanceName_ = value; + onChanged(); + return this; + } + + // optional string schemaName = 5; + private java.lang.Object schemaName_ = ""; + /** + * optional string schemaName = 5; + */ + public boolean hasSchemaName() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional string schemaName = 5; + */ + public java.lang.String getSchemaName() { + java.lang.Object ref = schemaName_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + schemaName_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string schemaName = 5; + */ + public com.google.protobuf.ByteString + getSchemaNameBytes() { + java.lang.Object ref = schemaName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + schemaName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string schemaName = 5; + */ + public Builder setSchemaName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + schemaName_ = value; + onChanged(); + return this; + } + /** + * optional string schemaName = 5; + */ + public Builder clearSchemaName() { + bitField0_ = (bitField0_ & ~0x00000010); + schemaName_ = getDefaultInstance().getSchemaName(); + onChanged(); + return this; + } + /** + * optional string schemaName = 5; + */ + public Builder setSchemaNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000010; + schemaName_ = value; + onChanged(); + return this; + } + + // optional string tableName = 6; + private java.lang.Object tableName_ = ""; + /** + * optional string tableName = 6; + */ + public boolean hasTableName() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional string tableName = 6; + */ + public java.lang.String getTableName() { + java.lang.Object ref = tableName_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + tableName_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string tableName = 6; + */ + public com.google.protobuf.ByteString + getTableNameBytes() { + java.lang.Object ref = tableName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + tableName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string tableName = 6; + */ + public Builder setTableName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + tableName_ = value; + onChanged(); + return this; + } + /** + * optional string tableName = 6; + */ + public Builder clearTableName() { + bitField0_ = (bitField0_ & ~0x00000020); + tableName_ = getDefaultInstance().getTableName(); + onChanged(); + return this; + } + /** + * optional string tableName = 6; + */ + public Builder setTableNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + tableName_ = value; + onChanged(); + return this; + } + + // optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + private org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType eventType_ = org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType.UPDATE; + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + public boolean hasEventType() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + public org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType getEventType() { + return eventType_; + } + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + public Builder setEventType(org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000040; + eventType_ = value; + onChanged(); + return this; + } + /** + * optional .org.apache.inlong.sort.formats.binlog.EventType eventType = 7 [default = UPDATE]; + */ + public Builder clearEventType() { + bitField0_ = (bitField0_ & ~0x00000040); + eventType_ = org.apache.inlong.sort.formats.binlog.InLongBinlog.EventType.UPDATE; + onChanged(); + return this; + } + + // optional int64 executeTime = 8; + private long executeTime_ ; + /** + * optional int64 executeTime = 8; + */ + public boolean hasExecuteTime() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional int64 executeTime = 8; + */ + public long getExecuteTime() { + return executeTime_; + } + /** + * optional int64 executeTime = 8; + */ + public Builder setExecuteTime(long value) { + bitField0_ |= 0x00000080; + executeTime_ = value; + onChanged(); + return this; + } + /** + * optional int64 executeTime = 8; + */ + public Builder clearExecuteTime() { + bitField0_ = (bitField0_ & ~0x00000080); + executeTime_ = 0L; + onChanged(); + return this; + } + + // optional int64 executeOrder = 9; + private long executeOrder_ ; + /** + * optional int64 executeOrder = 9; + */ + public boolean hasExecuteOrder() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + /** + * optional int64 executeOrder = 9; + */ + public long getExecuteOrder() { + return executeOrder_; + } + /** + * optional int64 executeOrder = 9; + */ + public Builder setExecuteOrder(long value) { + bitField0_ |= 0x00000100; + executeOrder_ = value; + onChanged(); + return this; + } + /** + * optional int64 executeOrder = 9; + */ + public Builder clearExecuteOrder() { + bitField0_ = (bitField0_ & ~0x00000100); + executeOrder_ = 0L; + onChanged(); + return this; + } + + // optional string transferIp = 10; + private java.lang.Object transferIp_ = ""; + /** + * optional string transferIp = 10; + */ + public boolean hasTransferIp() { + return ((bitField0_ & 0x00000200) == 0x00000200); + } + /** + * optional string transferIp = 10; + */ + public java.lang.String getTransferIp() { + java.lang.Object ref = transferIp_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + transferIp_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string transferIp = 10; + */ + public com.google.protobuf.ByteString + getTransferIpBytes() { + java.lang.Object ref = transferIp_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + transferIp_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string transferIp = 10; + */ + public Builder setTransferIp( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000200; + transferIp_ = value; + onChanged(); + return this; + } + /** + * optional string transferIp = 10; + */ + public Builder clearTransferIp() { + bitField0_ = (bitField0_ & ~0x00000200); + transferIp_ = getDefaultInstance().getTransferIp(); + onChanged(); + return this; + } + /** + * optional string transferIp = 10; + */ + public Builder setTransferIpBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000200; + transferIp_ = value; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.inlong.sort.formats.binlog.RowData) + } + + static { + defaultInstance = new RowData(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.inlong.sort.formats.binlog.RowData) + } + + public interface PairOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional string key = 1; + /** + * optional string key = 1; + */ + boolean hasKey(); + /** + * optional string key = 1; + */ + java.lang.String getKey(); + /** + * optional string key = 1; + */ + com.google.protobuf.ByteString + getKeyBytes(); + + // optional string value = 2; + /** + * optional string value = 2; + */ + boolean hasValue(); + /** + * optional string value = 2; + */ + java.lang.String getValue(); + /** + * optional string value = 2; + */ + com.google.protobuf.ByteString + getValueBytes(); + } + /** + * Protobuf type {@code org.apache.inlong.sort.formats.binlog.Pair} + */ + public static final class Pair extends + com.google.protobuf.GeneratedMessage + implements PairOrBuilder { + // Use Pair.newBuilder() to construct. + private Pair(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private Pair(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final Pair defaultInstance; + public static Pair getDefaultInstance() { + return defaultInstance; + } + + public Pair getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private Pair( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + key_ = input.readBytes(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + value_ = input.readBytes(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Pair_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Pair_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.class, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public Pair parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Pair(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // optional string key = 1; + public static final int KEY_FIELD_NUMBER = 1; + private java.lang.Object key_; + /** + * optional string key = 1; + */ + public boolean hasKey() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * optional string key = 1; + */ + public java.lang.String getKey() { + java.lang.Object ref = key_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + key_ = s; + } + return s; + } + } + /** + * optional string key = 1; + */ + public com.google.protobuf.ByteString + getKeyBytes() { + java.lang.Object ref = key_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + key_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional string value = 2; + public static final int VALUE_FIELD_NUMBER = 2; + private java.lang.Object value_; + /** + * optional string value = 2; + */ + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional string value = 2; + */ + public java.lang.String getValue() { + java.lang.Object ref = value_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + value_ = s; + } + return s; + } + } + /** + * optional string value = 2; + */ + public com.google.protobuf.ByteString + getValueBytes() { + java.lang.Object ref = value_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + value_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + key_ = ""; + value_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getKeyBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, getValueBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getKeyBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(2, getValueBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.apache.inlong.sort.formats.binlog.Pair} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.inlong.sort.formats.binlog.InLongBinlog.PairOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Pair_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Pair_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.class, org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.Builder.class); + } + + // Construct using org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + key_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + value_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.internal_static_org_apache_inlong_sort_formats_binlog_Pair_descriptor; + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair getDefaultInstanceForType() { + return org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.getDefaultInstance(); + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair build() { + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair buildPartial() { + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair result = new org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.key_ = key_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.value_ = value_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair) { + return mergeFrom((org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair other) { + if (other == org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair.getDefaultInstance()) return this; + if (other.hasKey()) { + bitField0_ |= 0x00000001; + key_ = other.key_; + onChanged(); + } + if (other.hasValue()) { + bitField0_ |= 0x00000002; + value_ = other.value_; + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.inlong.sort.formats.binlog.InLongBinlog.Pair) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // optional string key = 1; + private java.lang.Object key_ = ""; + /** + * optional string key = 1; + */ + public boolean hasKey() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * optional string key = 1; + */ + public java.lang.String getKey() { + java.lang.Object ref = key_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + key_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string key = 1; + */ + public com.google.protobuf.ByteString + getKeyBytes() { + java.lang.Object ref = key_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + key_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string key = 1; + */ + public Builder setKey( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + key_ = value; + onChanged(); + return this; + } + /** + * optional string key = 1; + */ + public Builder clearKey() { + bitField0_ = (bitField0_ & ~0x00000001); + key_ = getDefaultInstance().getKey(); + onChanged(); + return this; + } + /** + * optional string key = 1; + */ + public Builder setKeyBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + key_ = value; + onChanged(); + return this; + } + + // optional string value = 2; + private java.lang.Object value_ = ""; + /** + * optional string value = 2; + */ + public boolean hasValue() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional string value = 2; + */ + public java.lang.String getValue() { + java.lang.Object ref = value_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + value_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * optional string value = 2; + */ + public com.google.protobuf.ByteString + getValueBytes() { + java.lang.Object ref = value_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + value_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * optional string value = 2; + */ + public Builder setValue( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + return this; + } + /** + * optional string value = 2; + */ + public Builder clearValue() { + bitField0_ = (bitField0_ & ~0x00000002); + value_ = getDefaultInstance().getValue(); + onChanged(); + return this; + } + /** + * optional string value = 2; + */ + public Builder setValueBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + value_ = value; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:org.apache.inlong.sort.formats.binlog.Pair) + } + + static { + defaultInstance = new Pair(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:org.apache.inlong.sort.formats.binlog.Pair) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_inlong_sort_formats_binlog_Column_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_inlong_sort_formats_binlog_Column_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_inlong_sort_formats_binlog_RowData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_inlong_sort_formats_binlog_RowData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_org_apache_inlong_sort_formats_binlog_Pair_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_org_apache_inlong_sort_formats_binlog_Pair_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\022InLongBinlog.proto\022%org.apache.inlong." + + "sort.formats.binlog\"\310\001\n\006Column\022\r\n\005index\030" + + "\001 \001(\005\022\017\n\007sqlType\030\002 \001(\005\022\014\n\004name\030\003 \001(\t\022\r\n\005" + + "isKey\030\004 \001(\010\022\017\n\007updated\030\005 \001(\010\022\025\n\006isNull\030\006" + + " \001(\010:\005false\022:\n\005props\030\007 \003(\0132+.org.apache." + + "inlong.sort.formats.binlog.Pair\022\r\n\005value" + + "\030\010 \001(\t\022\016\n\006length\030\t \001(\005\"\231\003\n\007RowData\022D\n\rbe" + + "foreColumns\030\001 \003(\0132-.org.apache.inlong.so" + + "rt.formats.binlog.Column\022C\n\014afterColumns" + + "\030\002 \003(\0132-.org.apache.inlong.sort.formats.", + "binlog.Column\022:\n\005props\030\003 \003(\0132+.org.apach" + + "e.inlong.sort.formats.binlog.Pair\022\024\n\014ins" + + "tanceName\030\004 \001(\t\022\022\n\nschemaName\030\005 \001(\t\022\021\n\tt" + + "ableName\030\006 \001(\t\022K\n\teventType\030\007 \001(\01620.org." + + "apache.inlong.sort.formats.binlog.EventT" + + "ype:\006UPDATE\022\023\n\013executeTime\030\010 \001(\003\022\024\n\014exec" + + "uteOrder\030\t \001(\003\022\022\n\ntransferIp\030\n \001(\t\"\"\n\004Pa" + + "ir\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t*f\n\tEventT" + + "ype\022\n\n\006INSERT\020\001\022\n\n\006UPDATE\020\002\022\n\n\006DELETE\020\003\022" + + "\n\n\006CREATE\020\004\022\t\n\005ALTER\020\005\022\t\n\005ERASE\020\006\022\t\n\005QUE", + "RY\020\007\022\010\n\004GTID\020\010B7\n%org.apache.inlong.sort" + + ".formats.binlogB\014InLongBinlogH\001" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_org_apache_inlong_sort_formats_binlog_Column_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_org_apache_inlong_sort_formats_binlog_Column_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_inlong_sort_formats_binlog_Column_descriptor, + new java.lang.String[] { "Index", "SqlType", "Name", "IsKey", "Updated", "IsNull", "Props", "Value", "Length", }); + internal_static_org_apache_inlong_sort_formats_binlog_RowData_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_org_apache_inlong_sort_formats_binlog_RowData_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_inlong_sort_formats_binlog_RowData_descriptor, + new java.lang.String[] { "BeforeColumns", "AfterColumns", "Props", "InstanceName", "SchemaName", "TableName", "EventType", "ExecuteTime", "ExecuteOrder", "TransferIp", }); + internal_static_org_apache_inlong_sort_formats_binlog_Pair_descriptor = + getDescriptor().getMessageTypes().get(2); + internal_static_org_apache_inlong_sort_formats_binlog_Pair_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_org_apache_inlong_sort_formats_binlog_Pair_descriptor, + new java.lang.String[] { "Key", "Value", }); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowFormatInfo.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowFormatInfo.java index e55bd02e6b6..27b38459dfc 100644 --- a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowFormatInfo.java +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowFormatInfo.java @@ -74,7 +74,7 @@ public RowTypeInfo getTypeInfo() { fieldTypeInfos[i] = fieldFormatInfos[i].getTypeInfo(); } - return new RowTypeInfo(fieldNames, fieldTypeInfos); + return new RowTypeInfo(fieldNames, fieldTypeInfos, new String[fieldNames.length]); } private static void checkArity( diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowTypeInfo.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowTypeInfo.java index 353da76fe3c..1e0dd72b914 100644 --- a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowTypeInfo.java +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/common/RowTypeInfo.java @@ -34,9 +34,12 @@ public class RowTypeInfo implements TypeInfo { private static final String FIELD_FIELD_NAMES = "fieldNames"; private static final String FIELD_FIELD_TYPES = "fieldTypes"; + private static final String FIELD_FIELD_DESCRIPTIONS = "fieldDescriptions"; - public static final RowTypeInfo EMPTY = - new RowTypeInfo(new String[0], new TypeInfo[0]); + public static final RowTypeInfo EMPTY = new RowTypeInfo( + new String[0], + new TypeInfo[0], + new String[0]); @JsonProperty(FIELD_FIELD_NAMES) @Nonnull @@ -46,34 +49,41 @@ public class RowTypeInfo implements TypeInfo { @Nonnull private final TypeInfo[] fieldTypeInfos; + @JsonProperty(FIELD_FIELD_DESCRIPTIONS) + @Nonnull + private final String[] fieldDescriptions; + @JsonCreator public RowTypeInfo( @JsonProperty(FIELD_FIELD_NAMES) @Nonnull String[] fieldNames, - @JsonProperty(FIELD_FIELD_TYPES) @Nonnull TypeInfo[] fieldTypeInfos) { - checkArity(fieldNames, fieldTypeInfos); + @JsonProperty(FIELD_FIELD_TYPES) @Nonnull TypeInfo[] fieldTypeInfos, + @JsonProperty(FIELD_FIELD_DESCRIPTIONS) @Nonnull String[] fieldDescriptions) { + checkArity(fieldNames, fieldTypeInfos, fieldDescriptions); checkDuplicates(fieldNames); this.fieldNames = fieldNames; this.fieldTypeInfos = fieldTypeInfos; + this.fieldDescriptions = fieldDescriptions; } private static void checkArity( String[] fieldNames, - TypeInfo[] fieldTypeInfos) { + TypeInfo[] fieldTypeInfos, + String[] fieldDescriptions) { if (fieldNames.length != fieldTypeInfos.length) { - throw new IllegalArgumentException("The number of names and " + "formats is not equal."); + throw new IllegalArgumentException(String.format("The number of names and " + + "types is not equal. FieldNames: %s, filedTypes: %s", + Arrays.toString(fieldNames), Arrays.toString(fieldTypeInfos))); } - } - - private static void checkDuplicates(String[] fieldNames) { - long numFieldNames = fieldNames.length; - long numDistinctFieldNames = - Arrays.stream(fieldNames) - .collect(Collectors.toSet()) - .size(); - - if (numDistinctFieldNames != numFieldNames) { - throw new IllegalArgumentException("There exist duplicated " + "field names."); + if (fieldDescriptions.length != fieldTypeInfos.length) { + throw new IllegalArgumentException(String.format("The number of descriptions and " + + "types is not equal. FieldDescriptions: %s, filedTypes: %s", + Arrays.toString(fieldDescriptions), Arrays.toString(fieldTypeInfos))); + } + if (fieldDescriptions.length != fieldNames.length) { + throw new IllegalArgumentException(String.format("The number of descriptions and " + + "names is not equal. FieldDescriptions: %s, filedNames: %s", + Arrays.toString(fieldDescriptions), Arrays.toString(fieldNames))); } } @@ -87,6 +97,11 @@ public TypeInfo[] getFieldTypeInfos() { return fieldTypeInfos; } + @Nonnull + public String[] getFieldDescriptions() { + return fieldDescriptions; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -98,7 +113,9 @@ public boolean equals(Object o) { } RowTypeInfo that = (RowTypeInfo) o; - return Arrays.equals(fieldTypeInfos, that.fieldTypeInfos); + return Arrays.equals(fieldNames, that.fieldNames) && + Arrays.equals(fieldTypeInfos, that.fieldTypeInfos) && + Arrays.equals(fieldDescriptions, that.fieldDescriptions); } @Override @@ -108,6 +125,22 @@ public int hashCode() { @Override public String toString() { - return "RowTypeInfo{" + "fieldTypeInfos=" + Arrays.toString(fieldTypeInfos) + '}'; + return "RowTypeInfo{" + + "fieldNames=" + Arrays.toString(fieldNames) + + "fieldTypeInfos=" + Arrays.toString(fieldTypeInfos) + + "fieldDescriptions=" + Arrays.toString(fieldDescriptions) + + '}'; + } + + private static void checkDuplicates(String[] fieldNames) { + long numFieldNames = fieldNames.length; + long numDistinctFieldNames = + Arrays.stream(fieldNames) + .collect(Collectors.toSet()) + .size(); + + if (numDistinctFieldNames != numFieldNames) { + throw new IllegalArgumentException("There exist duplicated " + "field names."); + } } } diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/CallbackCollector.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/CallbackCollector.java new file mode 100644 index 00000000000..4f707c08b67 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/CallbackCollector.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.apache.flink.util.Collector; +import org.apache.flink.util.function.ThrowingConsumer; + +import static org.apache.flink.util.Preconditions.checkNotNull; +/** + * A collector supporting callback. It also hides the exception thrown in method declaration to make lambda expression + * shorter. + */ +public class CallbackCollector implements Collector { + + private final ThrowingConsumer callback; + + public CallbackCollector(ThrowingConsumer callback) { + this.callback = checkNotNull(callback); + } + + @Override + public void collect(T t) { + try { + callback.accept(t); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/FailureHandler.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/FailureHandler.java new file mode 100644 index 00000000000..00b0249d514 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/FailureHandler.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import java.io.Serializable; + +/** + * Interface to handle the failure on parsing InLongMsg data. + */ +public interface FailureHandler extends Serializable { + + /** + * This method is called when there is a failure occurred while parsing InLongMsg head. + * + * @param attribute the attribute which head is parsed from + * @param exception the thrown exception + * @throws Exception the exception + */ + void onParsingHeadFailure(String attribute, Exception exception) throws Exception; + + /** + * This method is called when there is a failure occurred while parsing InLongMsg body. + * + * @param body the body bytes which body is parsed from + * @param exception the thrown exception + * @throws Exception the exception + */ + void onParsingBodyFailure(byte[] body, Exception exception) throws Exception; + + /** + * This method is called when there is a failure occurred while converting head and body to row. + * + * @param head the head of row + * @param body the body of row + * @param exception the thrown exception + * @throws Exception the exception + */ + void onConvertingRowFailure(InLongMsgHead head, InLongMsgBody body, Exception exception) throws Exception; +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/IgnoreFailureHandler.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/IgnoreFailureHandler.java new file mode 100644 index 00000000000..33f46b1c2b5 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/IgnoreFailureHandler.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An implementation of {@link FailureHandler} that ignores the failure. + */ +public class IgnoreFailureHandler implements FailureHandler { + + private static final Logger LOG = LoggerFactory.getLogger(IgnoreFailureHandler.class); + + @Override + public void onParsingHeadFailure(String attribute, Exception exception) { + LOG.warn("Cannot properly parse the head {}", attribute, exception); + } + + @Override + public void onParsingBodyFailure(byte[] body, Exception exception) { + LOG.warn("Cannot properly parse the body.", exception); + } + + @Override + public void onConvertingRowFailure(InLongMsgHead head, InLongMsgBody body, Exception exception) { + LOG.warn("Cannot properly convert the InLongMsg ({}, {})", head, body, exception); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + return o != null && getClass() == o.getClass(); + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgBody.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgBody.java new file mode 100644 index 00000000000..f0860b1b9c4 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgBody.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * The body deserialized from {@link InLongMsgBody}. + */ +public class InLongMsgBody implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * The body of the record. + */ + private final byte[] data; + + /** + * The interface of the record. + */ + private final String tid; + + /** + * The fields extracted from the body. + */ + private final List fields; + + /** + * The entries extracted from the body. + */ + private final Map entries; + + public InLongMsgBody( + byte[] data, + String tid, + List fields, + Map entries) { + this.data = data; + this.tid = tid; + this.fields = fields; + this.entries = entries; + } + + public byte[] getData() { + return data; + } + + public String getTid() { + return tid; + } + + public List getFields() { + return fields; + } + + public Map getEntries() { + return entries; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + InLongMsgBody inLongMsgBody = (InLongMsgBody) o; + return Arrays.equals(data, inLongMsgBody.data); + } + + @Override + public int hashCode() { + return Arrays.hashCode(data); + } + + @Override + public String toString() { + return "InLongMsgBody{" + "data=" + Arrays.toString(data) + ", tid='" + tid + '\'' + + ", fields=" + fields + ", entries=" + entries + '}'; + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgHead.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgHead.java new file mode 100644 index 00000000000..61e6f6234f1 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgHead.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.apache.inlong.common.msg.InLongMsg; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * The head deserialized from {@link InLongMsg}. + */ +public class InLongMsgHead implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * The attributes in the head. + */ + private final Map attributes; + + /** + * The interface of the record. + */ + private final String tid; + + /** + * The time of the record. + */ + private final Timestamp time; + + /** + * The predefined fields extracted from the head. + */ + private final List predefinedFields; + + public InLongMsgHead( + Map attributes, + String tid, + Timestamp time, + List predefinedFields) { + this.attributes = attributes; + this.tid = tid; + this.time = time; + this.predefinedFields = predefinedFields; + } + + public Map getAttributes() { + return attributes; + } + + public String getTid() { + return tid; + } + + public Timestamp getTime() { + return time; + } + + public List getPredefinedFields() { + return predefinedFields; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + InLongMsgHead that = (InLongMsgHead) o; + return Objects.equals(attributes, that.attributes) + && Objects.equals(tid, that.tid) + && Objects.equals(time, that.time) + && Objects.equals(predefinedFields, that.predefinedFields); + } + + @Override + public int hashCode() { + return Objects.hash(attributes, tid, time, predefinedFields); + } + + @Override + public String toString() { + return "InLongMsgHead{" + + "attributes=" + attributes + + ", tid='" + tid + '\'' + + ", time=" + time + + ", predefinedFields=" + predefinedFields + + '}'; + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMetadata.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMetadata.java new file mode 100644 index 00000000000..f5a1eca4278 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMetadata.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.types.DataType; + +/** + * Metadata of InLongMsg. + */ +public class InLongMsgMetadata { + + /** + * ReadableMetadata for InLongMsg. + */ + public enum ReadableMetadata { + + TID("metadata-tid", DataTypes.STRING()); + + final String key; + + final DataType dataType; + + ReadableMetadata(String key, DataType dataType) { + this.key = key; + this.dataType = dataType; + } + + public String getKey() { + return key; + } + + public DataType getDataType() { + return dataType; + } + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMixedValidator.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMixedValidator.java new file mode 100644 index 00000000000..cd4477c6fec --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMixedValidator.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.apache.inlong.sort.formats.base.TableFormatConstants; + +import org.apache.flink.table.descriptors.DescriptorProperties; +import org.apache.flink.table.descriptors.DescriptorValidator; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_PROPERTY_VERSION; + +/** + * Validator for mixed inlongmsg formats. + */ +public class InLongMsgMixedValidator implements DescriptorValidator { + + @Override + public void validate(DescriptorProperties properties) { + getValidateProperties(properties); + properties.validateString(TableFormatConstants.FORMAT_DELIMITER, true, 1, 1); + properties.validateString(TableFormatConstants.FORMAT_ENTRY_DELIMITER, true, 1, 1); + properties.validateString(TableFormatConstants.FORMAT_KV_DELIMITER, true, 1, 1); + properties.validateString(TableFormatConstants.FORMAT_QUOTE_CHARACTER, true, 1, 1); + properties.validateBoolean(TableFormatConstants.FORMAT_IGNORE_ERRORS, true); + } + + public static void getValidateProperties(DescriptorProperties properties) { + properties.validateString(TableFormatConstants.FORMAT_TYPE, false, 1); + properties.validateString(FORMAT_PROPERTY_VERSION, true, 1); + properties.validateString(TableFormatConstants.FORMAT_ESCAPE_CHARACTER, true, 1, 1); + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgOptions.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgOptions.java new file mode 100644 index 00000000000..e07b1022223 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgOptions.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.api.ValidationException; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_CHARSET; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_DELETE_HEAD_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_INCLUDE_UPDATE_BEFORE; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_METADATA_FIELD_NAME; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_RETAIN_PREDEFINED_FIELD; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_ATTRIBUTE_FIELD_NAME; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_CHARSET; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_DELETE_HEAD_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_ESCAPE_CHARACTER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_FIELD_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_INCLUDE_UPDATE_BEFORE; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_LINE_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_METADATA_FIELD_NAME; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_NULL_LITERAL; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_QUOTE_CHARACTER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_RETAIN_PREDEFINED_FIELD; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_TIME_FIELD_NAME; + +public class InLongMsgOptions { + + private InLongMsgOptions() { + } + + public static final ConfigOption INNER_FORMAT = + ConfigOptions.key("inner.format") + .stringType() + .noDefaultValue() + .withDescription("Defines the format identifier for encoding attr data. \n" + + "The identifier is used to discover a suitable format factory."); + + public static final ConfigOption IGNORE_PARSE_ERRORS = + ConfigOptions.key("ignore-parse-errors") + .booleanType() + .defaultValue(false) + .withDescription("Optional flag to skip fields and rows with parse errors instead of failing;\n" + + "fields are set to null in case of errors"); + + public static void validateDecodingFormatOptions(ReadableConfig config) { + String innerFormat = config.get(INNER_FORMAT); + if (innerFormat == null) { + throw new ValidationException( + INNER_FORMAT.key() + " shouldn't be null."); + } + } + + public static final ConfigOption TIME_FIELD_NAME = + ConfigOptions.key(FORMAT_TIME_FIELD_NAME) + .stringType() + .noDefaultValue() + .withDescription("The name of the time field in InLongMsg."); + + public static final ConfigOption ATTRIBUTE_FIELD_NAME = + ConfigOptions.key(FORMAT_ATTRIBUTE_FIELD_NAME) + .stringType() + .noDefaultValue() + .withDescription("The name of the attribute field in InLongMsg)"); + + public static final ConfigOption CHARSET = + ConfigOptions.key(FORMAT_CHARSET) + .stringType() + .defaultValue(DEFAULT_CHARSET) + .withDescription("Optional text encoding format ('utf-8' by default)"); + + public static final ConfigOption FIELD_DELIMITER = + ConfigOptions.key(FORMAT_FIELD_DELIMITER) + .stringType() + .defaultValue(String.valueOf(DEFAULT_DELIMITER)) + .withDescription("Optional field delimiter character (',' by default)"); + + public static final ConfigOption LINE_DELIMITER = + ConfigOptions.key(FORMAT_LINE_DELIMITER) + .stringType() + .noDefaultValue() + .withDescription( + "Optional line delimiter character"); + + public static final ConfigOption ESCAPE_CHARACTER = + ConfigOptions.key(FORMAT_ESCAPE_CHARACTER) + .stringType() + .noDefaultValue() + .withDescription( + "Optional escape character for escaping values (disabled by default)"); + + public static final ConfigOption QUOTE_CHARACTER = + ConfigOptions.key(FORMAT_QUOTE_CHARACTER) + .stringType() + .noDefaultValue() + .withDescription( + "Optional quote character for enclosing field values (disabled by default)"); + + public static final ConfigOption NULL_LITERAL = + ConfigOptions.key(FORMAT_NULL_LITERAL) + .stringType() + .noDefaultValue() + .withDescription( + "Optional null literal string that is interpreted as a\n" + + "null value (disabled by default)"); + + public static final ConfigOption IGNORE_ERRORS = + ConfigOptions.key(FORMAT_IGNORE_ERRORS) + .booleanType() + .defaultValue(DEFAULT_IGNORE_ERRORS) + .withDescription( + "Optional flag to skip fields and rows with parse errors instead of failing;\n" + + "fields are set to null in case of errors"); + + public static final ConfigOption DELETE_HEAD_DELIMITER = + ConfigOptions.key(FORMAT_DELETE_HEAD_DELIMITER) + .booleanType() + .defaultValue(DEFAULT_DELETE_HEAD_DELIMITER) + .withDescription( + "True if the head delimiter should be removed (false by default)."); + + public static final ConfigOption RETAIN_PREDEFINED_FIELD = + ConfigOptions.key(FORMAT_RETAIN_PREDEFINED_FIELD) + .booleanType() + .defaultValue(DEFAULT_RETAIN_PREDEFINED_FIELD) + .withDescription( + "True if the retain predefined field should be skip the predefined Field. (true by default)."); + + public static final ConfigOption METADATA_FIELD_NAME = + ConfigOptions.key(FORMAT_METADATA_FIELD_NAME) + .stringType() + .defaultValue(DEFAULT_METADATA_FIELD_NAME) + .withDescription( + "True if the retain predefined field should be skip the predefined Field. (true by default)."); + + public static final ConfigOption INCLUDE_UPDATE_BEFORE = + ConfigOptions.key(FORMAT_INCLUDE_UPDATE_BEFORE) + .booleanType() + .defaultValue(DEFAULT_INCLUDE_UPDATE_BEFORE) + .withDescription( + "True if the retain predefined field should be skip the predefined Field. (true by default)."); +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgValidator.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgValidator.java new file mode 100644 index 00000000000..e33071d1716 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgValidator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.apache.inlong.sort.formats.base.FormatDescriptorValidator; + +import org.apache.flink.table.descriptors.DescriptorProperties; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_ENTRY_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_ESCAPE_CHARACTER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_KV_DELIMITER; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_QUOTE_CHARACTER; +import static org.apache.inlong.sort.formats.base.TableFormatUtils.validateSchema; + +/** + * Validator for mixed inlongmsg formats. + */ +public class InLongMsgValidator extends FormatDescriptorValidator { + + @Override + public void validate(DescriptorProperties properties) { + super.validate(properties); + + properties.validateString(FORMAT_DELIMITER, true, 1, 1); + properties.validateString(FORMAT_ENTRY_DELIMITER, true, 1, 1); + properties.validateString(FORMAT_KV_DELIMITER, true, 1, 1); + properties.validateString(FORMAT_ESCAPE_CHARACTER, true, 1, 1); + properties.validateString(FORMAT_QUOTE_CHARACTER, true, 1, 1); + properties.validateBoolean(FORMAT_IGNORE_ERRORS, true); + + validateSchema(properties); + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgWrap.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgWrap.java new file mode 100644 index 00000000000..fba2731142f --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgWrap.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import java.io.Serializable; +import java.util.List; + +/** + * The body deserialized from {@link InLongMsgWrap}. + */ +public class InLongMsgWrap implements Serializable { + + private final InLongMsgHead inLongMsgHead; + + private final List inLongMsgBodyList; + + public InLongMsgWrap(InLongMsgHead inLongMsgHead, List inLongMsgBodyList) { + this.inLongMsgHead = inLongMsgHead; + this.inLongMsgBodyList = inLongMsgBodyList; + } + + public InLongMsgHead getInLongMsgHead() { + return inLongMsgHead; + } + + public List getInLongMsgBodyList() { + return inLongMsgBodyList; + } +} \ No newline at end of file diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/NoOpFailureHandler.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/NoOpFailureHandler.java new file mode 100644 index 00000000000..3f4eb090faf --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/inlongmsg/NoOpFailureHandler.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsg; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An implementation of {@link FailureHandler} that just throws the exception out. + */ +public class NoOpFailureHandler implements FailureHandler { + + private static final Logger LOG = LoggerFactory.getLogger(NoOpFailureHandler.class); + + @Override + public void onParsingHeadFailure(String attribute, Exception exception) throws Exception { + LOG.error("Cannot properly parse the head {}", attribute, exception); + throw exception; + } + + @Override + public void onParsingBodyFailure(byte[] body, Exception exception) throws Exception { + LOG.error("Cannot properly parse the body: {}.", new String(body), exception); + throw exception; + } + + @Override + public void onConvertingRowFailure(InLongMsgHead head, InLongMsgBody body, Exception exception) throws Exception { + LOG.error("Cannot properly convert the InLongMsg ({}, {})", head, body, exception); + throw exception; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + return o != null && getClass() == o.getClass(); + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/CommonUtils.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/CommonUtils.java new file mode 100644 index 00000000000..db0963ac05a --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/CommonUtils.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.util; + +import org.apache.flink.util.Collector; + +import java.util.ArrayList; +import java.util.List; + +/** + * Common utils. + */ +public class CommonUtils { + + /** + * A simple collector which collects elements into a list. + * @param Type of elements to collect. + */ + public static class SimpleListCollector implements Collector { + + private List elements = new ArrayList<>(); + + @Override + public void collect(T element) { + elements.add(element); + } + + @Override + public void close() { + + } + + public List getElements() { + return elements; + } + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/StringUtils.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/StringUtils.java index e46fda45796..5ba0c248f41 100644 --- a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/StringUtils.java +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/StringUtils.java @@ -39,21 +39,8 @@ public class StringUtils { private static final int STATE_QUOTING = 16; /** - * Splits the kv text. - * - *

Both escaping and quoting is supported. When the escape character is - * not '\0', then the next character to the escape character will be - * escaped. When the quote character is not '\0', then all characters - * between consecutive quote characters will be escaped.

- * - * @param text The text to be split. - * @param entryDelimiter The delimiter of entries. - * @param kvDelimiter The delimiter between key and value. - * @param escapeChar The escaping character. Only valid if not '\0'. - * @param quoteChar The quoting character. - * @return The fields split from the text. + * @see StringUtils#splitKv(String, Character, Character, Character,Character, Character) */ - @SuppressWarnings("checkstyle:MissingSwitchDefault") public static Map splitKv( @Nonnull String text, @Nonnull Character entryDelimiter, @@ -241,9 +228,10 @@ public static String concatKv( @Nonnull Character kvDelimiter, @Nullable Character escapeChar, @Nullable Character quoteChar) { - if (fieldKeys.length != fieldValues.length) { - throw new IllegalArgumentException("The keys' number " + fieldKeys.length - + " doesn't match values' number " + fieldValues.length); + if (fieldKeys.length < fieldValues.length) { + throw new IllegalArgumentException("The keys' number " + + fieldKeys.length + " is less than values' number " + + fieldValues.length); } Collection delimiters = @@ -295,8 +283,9 @@ private static void encodeText( stringBuilder.append(ch); stringBuilder.append(quoteChar); } else { - throw new IllegalArgumentException("There is a delimiter in the text, " - + "but neither escape nor quote character is specified."); + throw new IllegalArgumentException("There is a delimiter " + + "in the text, but neither escape nor quote character " + + "is specified."); } } else if (escapeChar != null && ch == escapeChar) { stringBuilder.append(escapeChar); @@ -306,8 +295,9 @@ private static void encodeText( stringBuilder.append(escapeChar); stringBuilder.append(ch); } else { - throw new IllegalArgumentException("There is a quote character in the text, " - + "but escape character is not specified."); + throw new IllegalArgumentException("There is a quote " + + "character in the text, but escape character is not " + + "specified."); } } else { stringBuilder.append(ch); @@ -319,12 +309,6 @@ private static void encodeText( * Splits a single line of csv text. * * @see StringUtils#splitCsv(String, Character, Character, Character, Character, boolean) - * - * @param text The text to be split. - * @param delimiter The delimiter of fields. - * @param escapeChar The escaping character. Only valid if not '\0'. - * @param quoteChar The quoting character. - * @return The split array content. */ public static String[] splitCsv( @Nonnull String text, @@ -340,13 +324,6 @@ public static String[] splitCsv( /** * @see StringUtils#splitCsv(String, Character, Character, Character, Character, boolean) - * - * @param text The text to be split. - * @param delimiter The delimiter of fields. - * @param escapeChar The escaping character. Only valid if not '\0'. - * @param quoteChar The quoting character. - * @param lineDelimiter The delimiter between lines, e.g. '\n'. - * @return The split value. */ public static String[][] splitCsv( @Nonnull String text, @@ -510,9 +487,9 @@ public static String concatCsv( for (int i = 0; i < field.length(); ++i) { char ch = field.charAt(i); - if (ch == delimiter - || (escapeChar != null && ch == escapeChar) - || (quoteChar != null && ch == quoteChar)) { + if (ch == delimiter || + (escapeChar != null && ch == escapeChar) || + (quoteChar != null && ch == quoteChar)) { if (escapeChar != null) { stringBuilder.append(escapeChar); @@ -522,8 +499,10 @@ public static String concatCsv( stringBuilder.append(ch); stringBuilder.append(quoteChar); } else { - throw new IllegalArgumentException("There exist special characters in the text, " - + "but neither escape character nor quote character is configured."); + throw new IllegalArgumentException("There exist " + + "special characters in the text but neither " + + "escape character nor quote character is " + + "configured."); } } else { stringBuilder.append(ch); diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/ValidateUtils.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/ValidateUtils.java new file mode 100644 index 00000000000..96f3c366435 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/util/ValidateUtils.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.util; + +import org.apache.flink.configuration.ReadableConfig; + +/** + * Used for validate format options {@link ReadableConfig}. + */ +public class ValidateUtils { + + public static void validateString( + String key, + String value, + boolean isOptional) { + validateString(key, value, isOptional, 0, Integer.MAX_VALUE); + } + + public static void validateString( + String key, + String value, + boolean isOptional, + int minLen) { + validateString(key, value, isOptional, minLen, Integer.MAX_VALUE); + } + + public static void validateString( + String key, + String value, + boolean isOptional, + int minLen, + int maxLen) { + + if (value == null) { + if (!isOptional) { + throw new IllegalArgumentException(String.format("Value of key %s must not be null!", key)); + } + } else { + if (value.length() < minLen) { + throw new IllegalArgumentException( + String.format("Value's length of key %s should be shorter than %d!", + key, minLen)); + } + + if (value.length() > maxLen) { + throw new IllegalArgumentException( + String.format("Value's length of key %s should be longer than %d!", + key, minLen)); + } + } + } +} diff --git a/inlong-sort/sort-formats/format-common/src/main/proto/InLongBinlog.proto b/inlong-sort/sort-formats/format-common/src/main/proto/InLongBinlog.proto new file mode 100644 index 00000000000..363fb62d687 --- /dev/null +++ b/inlong-sort/sort-formats/format-common/src/main/proto/InLongBinlog.proto @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax="proto2"; +package org.apache.inlong.sort.formats.binlog; +option java_package = "org.apache.inlong.sort.formats.binlog"; +option java_outer_classname = "InLongBinlog"; +option optimize_for = SPEED; + +/**************************************************************** + * message model + ****************************************************************/ + +message Column { + optional int32 index = 1; + + optional int32 sqlType = 2; + + optional string name = 3; + + optional bool isKey = 4; + + optional bool updated = 5; + + optional bool isNull = 6 [default = false]; + + repeated Pair props = 7; + + optional string value = 8; + + optional int32 length = 9; +} + +message RowData { + + repeated Column beforeColumns = 1; + + repeated Column afterColumns = 2; + + repeated Pair props = 3; + + optional string instanceName = 4; + + optional string schemaName = 5; + + optional string tableName = 6; + + optional EventType eventType = 7 [default = UPDATE]; + + optional int64 executeTime = 8; + + optional int64 executeOrder = 9; + + optional string transferIp = 10; +} + +message Pair{ + optional string key = 1; + optional string value = 2; +} + +enum EventType { + INSERT = 1; + UPDATE = 2; + DELETE = 3; + CREATE = 4; + ALTER = 5; + ERASE = 6; + QUERY = 7; + GTID = 8; +} \ No newline at end of file diff --git a/inlong-sort/sort-formats/format-row/format-base/pom.xml b/inlong-sort/sort-formats/format-row/format-base/pom.xml index dbcdd4ff6dc..90c65108415 100644 --- a/inlong-sort/sort-formats/format-row/format-base/pom.xml +++ b/inlong-sort/sort-formats/format-row/format-base/pom.xml @@ -38,7 +38,6 @@ org.apache.inlong sort-format-common ${project.version} - provided diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatConstants.java b/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatConstants.java deleted file mode 100644 index eb662aa8ce5..00000000000 --- a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatConstants.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.inlong.sort.formats.base; - -/** - * Common constants used in various formats. - */ -public class TableFormatConstants { - - public static final String FORMAT_SCHEMA = "format.schema"; - public static final String FORMAT_DELIMITER = "format.delimiter"; - public static final String FORMAT_DERIVE_SCHEMA = "format.derive-schema"; - public static final String FORMAT_ENTRY_DELIMITER = "format.entry-delimiter"; - public static final String FORMAT_KV_DELIMITER = "format.kv-delimiter"; - public static final String FORMAT_LINE_DELIMITER = "format.line-delimiter"; - public static final String FORMAT_NULL_LITERAL = "format.null-literal"; - public static final String FORMAT_ESCAPE_CHARACTER = "format.escape-character"; - public static final String FORMAT_QUOTE_CHARACTER = "format.quote-character"; - public static final String FORMAT_IGNORE_ERRORS = "format.ignore-errors"; - public static final String FORMAT_CHARSET = "format.charset"; - public static final String FORMAT_TYPE = "format.type"; - public static final String FORMAT_PROPERTY_VERSION = "format.property-version"; - - public static final char DEFAULT_DELIMITER = ','; - public static final char DEFAULT_ENTRY_DELIMITER = '&'; - public static final char DEFAULT_KV_DELIMITER = '='; - public static final Character DEFAULT_LINE_DELIMITER = null; - public static final Character DEFAULT_ESCAPE_CHARACTER = null; - public static final Character DEFAULT_QUOTE_CHARACTER = null; - public static final String DEFAULT_NULL_LITERAL = null; - public static final boolean DEFAULT_IGNORE_ERRORS = false; - public static final String DEFAULT_CHARSET = "UTF-8"; -} diff --git a/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatForRowUtils.java b/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatForRowUtils.java new file mode 100644 index 00000000000..e81f9be3478 --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-base/src/main/java/org/apache/inlong/sort/formats/base/TableFormatForRowUtils.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.base; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.table.factories.TableFactoryService; +import org.apache.flink.types.Row; + +import java.util.Map; + +public class TableFormatForRowUtils extends TableFormatUtils { + + /** + * Returns the {@link DeserializationSchema} described by the given + * properties. + * + * @param properties The properties describing the deserializer. + * @param fields The fields to project. + * @param classLoader The class loader for the deserializer. + * @param The type of the data. + * @return The {@link DeserializationSchema} described by the properties. + */ + public static DeserializationSchema getProjectedDeserializationSchema( + final Map properties, + final int[] fields, + final ClassLoader classLoader) { + final ProjectedDeserializationSchemaFactory deserializationSchemaFactory = + TableFactoryService.find( + ProjectedDeserializationSchemaFactory.class, + properties, + classLoader); + + return deserializationSchemaFactory + .createProjectedDeserializationSchema(properties, fields); + } + + /** + * Returns the {@link SerializationSchema} described by the given + * properties. + * + * @param properties The properties describing the serializer. + * @param fields The fields to project. + * @param classLoader The class loader for the serializer. + * @return The {@link SerializationSchema} described by the properties. + */ + public static SerializationSchema getProjectedSerializationSchema( + final Map properties, + final int[] fields, + final ClassLoader classLoader) { + final ProjectedSerializationSchemaFactory serializationSchemaFactory = + TableFactoryService.find( + ProjectedSerializationSchemaFactory.class, + properties, + classLoader); + + return serializationSchemaFactory + .createProjectedSerializationSchema(properties, fields); + } + + /** + * Returns the {@link TableFormatSerializer} described by the given + * properties. + * + * @param properties The properties describing the serializer. + * @param classLoader The class loader for the serializer. + * @return The {@link TableFormatSerializer} described by the properties. + */ + public static TableFormatSerializer getTableFormatSerializer( + final Map properties, + final ClassLoader classLoader) { + final TableFormatSerializerFactory tableFormatSerializerFactory = + TableFactoryService.find( + TableFormatSerializerFactory.class, + properties, + classLoader); + + return tableFormatSerializerFactory + .createFormatSerializer(properties); + } + + /** + * Returns the {@link TableFormatDeserializer} described by the + * given properties. + * + * @param properties The properties describing the deserializer. + * @param classLoader The class loader for the deserializer. + * @return The {@link TableFormatDeserializer} described by the properties. + */ + public static TableFormatDeserializer getTableFormatDeserializer( + final Map properties, + final ClassLoader classLoader) { + final TableFormatDeserializerFactory tableFormatDeserializerFactory = + TableFactoryService.find( + TableFormatDeserializerFactory.class, + properties, + classLoader); + + return tableFormatDeserializerFactory + .createFormatDeserializer(properties); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvDeserializationSchema.java b/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvDeserializationSchema.java index 7e20070cf6a..f070cfef175 100644 --- a/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvDeserializationSchema.java +++ b/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvDeserializationSchema.java @@ -18,7 +18,7 @@ package org.apache.inlong.sort.formats.csv; import org.apache.inlong.sort.formats.base.DefaultDeserializationSchema; -import org.apache.inlong.sort.formats.base.TableFormatUtils; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.inlong.sort.formats.base.util.LogCounter; import org.apache.inlong.sort.formats.common.FormatInfo; import org.apache.inlong.sort.formats.common.RowFormatInfo; @@ -123,7 +123,7 @@ public CsvDeserializationSchema( @SuppressWarnings("unchecked") @Override public TypeInformation getProducedType() { - return (TypeInformation) TableFormatUtils.getType(rowFormatInfo.getTypeInfo()); + return (TypeInformation) TableFormatForRowUtils.getType(rowFormatInfo.getTypeInfo()); } @Override diff --git a/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvValidator.java b/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvValidator.java index 3c27e87649b..f39468fac4e 100644 --- a/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvValidator.java +++ b/inlong-sort/sort-formats/format-row/format-csv/src/main/java/org/apache/inlong/sort/formats/csv/CsvValidator.java @@ -18,7 +18,7 @@ package org.apache.inlong.sort.formats.csv; import org.apache.inlong.sort.formats.base.TableFormatConstants; -import org.apache.inlong.sort.formats.base.TableFormatUtils; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.flink.table.descriptors.DescriptorProperties; import org.apache.flink.table.descriptors.DescriptorValidator; @@ -30,10 +30,10 @@ public class CsvValidator implements DescriptorValidator { @Override public void validate(DescriptorProperties properties) { - TableFormatUtils.getValidateProperties(properties); + TableFormatForRowUtils.getValidateProperties(properties); properties.validateString(TableFormatConstants.FORMAT_DELIMITER, true, 1, 1); properties.validateString(TableFormatConstants.FORMAT_QUOTE_CHARACTER, true, 1, 1); - TableFormatUtils.validateSchema(properties); + TableFormatForRowUtils.validateSchema(properties); } } diff --git a/inlong-sort/sort-formats/format-row/format-csv/src/test/java/org/apache/inlong/sort/formats/csv/CsvFormatFactoryTest.java b/inlong-sort/sort-formats/format-row/format-csv/src/test/java/org/apache/inlong/sort/formats/csv/CsvFormatFactoryTest.java index 78f5a19b559..44c16eec25b 100644 --- a/inlong-sort/sort-formats/format-row/format-csv/src/test/java/org/apache/inlong/sort/formats/csv/CsvFormatFactoryTest.java +++ b/inlong-sort/sort-formats/format-row/format-csv/src/test/java/org/apache/inlong/sort/formats/csv/CsvFormatFactoryTest.java @@ -20,8 +20,8 @@ import org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer; import org.apache.inlong.sort.formats.base.DefaultTableFormatSerializer; import org.apache.inlong.sort.formats.base.TableFormatDeserializer; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.inlong.sort.formats.base.TableFormatSerializer; -import org.apache.inlong.sort.formats.base.TableFormatUtils; import org.apache.inlong.sort.formats.common.DateFormatInfo; import org.apache.inlong.sort.formats.common.FormatInfo; import org.apache.inlong.sort.formats.common.IntFormatInfo; @@ -107,7 +107,7 @@ public void testCreateTableFormatDeserializer() throws Exception { new DefaultTableFormatDeserializer(deserializationSchema); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); @@ -130,7 +130,7 @@ public void testCreateTableFormatDeserializerWithDerivation() { new DefaultTableFormatDeserializer(deserializationSchema); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); @@ -164,7 +164,7 @@ public void testCreateTableFormatSerializer() throws Exception { new DefaultTableFormatSerializer(serializationSchema); final TableFormatSerializer actualSer = - TableFormatUtils.getTableFormatSerializer( + TableFormatForRowUtils.getTableFormatSerializer( properties, getClass().getClassLoader()); @@ -187,7 +187,7 @@ public void testCreateTableFormatSerializerWithDerivation() { new DefaultTableFormatSerializer(serializationSchema); final TableFormatSerializer actualSer = - TableFormatUtils.getTableFormatSerializer( + TableFormatForRowUtils.getTableFormatSerializer( properties, getClass().getClassLoader()); @@ -218,7 +218,7 @@ public void testCreateProjectedDeserializationSchema() throws IOException { false); final DeserializationSchema actualDeser = - TableFormatUtils.getProjectedDeserializationSchema( + TableFormatForRowUtils.getProjectedDeserializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); @@ -239,7 +239,7 @@ public void testCreateProjectedDeserializationSchemaWithDerivation() { new CsvDeserializationSchema(TEST_FORMAT_SCHEMA); final DeserializationSchema actualDeser = - TableFormatUtils.getProjectedDeserializationSchema( + TableFormatForRowUtils.getProjectedDeserializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); @@ -271,7 +271,7 @@ public void testCreateProjectedSerializationSchema() throws IOException { false); final SerializationSchema actualSer = - TableFormatUtils.getProjectedSerializationSchema( + TableFormatForRowUtils.getProjectedSerializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); @@ -292,7 +292,7 @@ public void testCreateProjectedSerializationSchemaWithDerivation() { new CsvSerializationSchema(TEST_FORMAT_SCHEMA); final SerializationSchema actualSer = - TableFormatUtils.getProjectedSerializationSchema( + TableFormatForRowUtils.getProjectedSerializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-base/pom.xml b/inlong-sort/sort-formats/format-row/format-inlongmsg-base/pom.xml index a05114d8034..e35043d522d 100644 --- a/inlong-sort/sort-formats/format-row/format-inlongmsg-base/pom.xml +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-base/pom.xml @@ -45,14 +45,12 @@ org.apache.inlong sort-format-common ${project.version} - provided org.apache.inlong sort-format-base ${project.version} - provided diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMixedValidator.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMixedValidator.java index b68b0e697f4..bf14669bd1f 100644 --- a/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMixedValidator.java +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgMixedValidator.java @@ -18,7 +18,7 @@ package org.apache.inlong.sort.formats.inlongmsg; import org.apache.inlong.sort.formats.base.TableFormatConstants; -import org.apache.inlong.sort.formats.base.TableFormatUtils; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.flink.table.descriptors.DescriptorProperties; import org.apache.flink.table.descriptors.DescriptorValidator; @@ -30,7 +30,7 @@ public class InLongMsgMixedValidator implements DescriptorValidator { @Override public void validate(DescriptorProperties properties) { - TableFormatUtils.getValidateProperties(properties); + TableFormatForRowUtils.getValidateProperties(properties); properties.validateString(TableFormatConstants.FORMAT_DELIMITER, true, 1, 1); properties.validateString(TableFormatConstants.FORMAT_ENTRY_DELIMITER, true, 1, 1); properties.validateString(TableFormatConstants.FORMAT_KV_DELIMITER, true, 1, 1); diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgUtils.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgUtils.java index e8b3418614b..969b7fc19eb 100644 --- a/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgUtils.java +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-base/src/main/java/org/apache/inlong/sort/formats/inlongmsg/InLongMsgUtils.java @@ -19,7 +19,7 @@ import org.apache.inlong.common.msg.InLongMsg; import org.apache.inlong.sort.formats.base.TableFormatConstants; -import org.apache.inlong.sort.formats.base.TableFormatUtils; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.inlong.sort.formats.common.FormatInfo; import org.apache.inlong.sort.formats.common.RowFormatInfo; import org.apache.inlong.sort.formats.util.StringUtils; @@ -111,7 +111,7 @@ public static RowFormatInfo getDataFormatInfo( for (int i = 0; i < dataFieldNames.length; ++i) { dataFieldNames[i] = fieldNames[i + 2]; dataFieldFormatInfos[i] = - TableFormatUtils.deriveFormatInfo(fieldTypes[i + 2].getLogicalType()); + TableFormatForRowUtils.deriveFormatInfo(fieldTypes[i + 2].getLogicalType()); } return new RowFormatInfo(dataFieldNames, dataFieldFormatInfos); @@ -356,7 +356,7 @@ public static TypeInformation buildRowType( fieldTypes[1] = Types.MAP(Types.STRING, Types.STRING); for (int i = 0; i < dataFieldFormatInfos.length; ++i) { - fieldTypes[i + 2] = TableFormatUtils.getType(dataFieldFormatInfos[i].getTypeInfo()); + fieldTypes[i + 2] = TableFormatForRowUtils.getType(dataFieldFormatInfos[i].getTypeInfo()); } return Types.ROW_NAMED(fieldNames, fieldTypes); @@ -383,7 +383,7 @@ public static TypeInformation decorateRowTypeWithNeededHeadFields( @Nullable String timeFieldName, @Nullable String attributesFieldName, RowFormatInfo dataRowFormatInfo) { - RowTypeInfo rowTypeInfo = (RowTypeInfo) TableFormatUtils.getType(dataRowFormatInfo.getTypeInfo()); + RowTypeInfo rowTypeInfo = (RowTypeInfo) TableFormatForRowUtils.getType(dataRowFormatInfo.getTypeInfo()); return InLongMsgUtils.decorateRowTypeWithNeededHeadFields( timeFieldName, diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/pom.xml b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/pom.xml new file mode 100644 index 00000000000..095dc098428 --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/pom.xml @@ -0,0 +1,141 @@ + + + + 4.0.0 + + org.apache.inlong + format-row + 1.11.0-SNAPSHOT + + + sort-format-inlongmsg-binlog + jar + Apache InLong - Sort Format-InLongMsg-Binlog + + + ${project.parent.parent.parent.parent.basedir} + 2.5.0 + + + + + org.apache.inlong + sort-format-inlongmsg-base + ${project.version} + + + + org.apache.flink + flink-table-common + ${flink.version} + provided + + + + com.google.protobuf + protobuf-java + ${binlog.protobuf.version} + jar + + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-table-planner_${scala.binary.version} + ${flink.version} + test + + + + + + + + + + + + + + + + + org.apache.flink + flink-table-planner_${scala.binary.version} + ${flink.version} + test-jar + test + + + + + + japicmp-report + + + japicmp-report + + + + + + com.github.siom79.japicmp + japicmp-maven-plugin + + + false + false + + + + + + + + japicmp-check + + + !japicmp-report + + + + + + com.github.siom79.japicmp + japicmp-maven-plugin + + + + + + + + + + + diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlog.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlog.java new file mode 100644 index 00000000000..7875f526ac7 --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlog.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.base.FormatDescriptor; +import org.apache.inlong.sort.formats.common.FormatInfo; +import org.apache.inlong.sort.formats.common.FormatUtils; +import org.apache.inlong.sort.formats.common.RowFormatInfo; + +import org.apache.flink.table.descriptors.DescriptorProperties; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_SCHEMA; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.FORMAT_METADATA_FIELD_NAME; + +/** + * Format descriptor for InLong binlog. + * + */ +public class InLongMsgBinlog extends FormatDescriptor { + + public static final String FORMAT_TYPE_VALUE = "InLongMsg-Binlog"; + + private DescriptorProperties internalProperties = new DescriptorProperties(true); + + private final List fieldNames = new ArrayList<>(); + private final List fieldFormatInfos = new ArrayList<>(); + + public InLongMsgBinlog() { + super(FORMAT_TYPE_VALUE, 1); + } + + /** + * Ignores the errors in the serialization and deserialization. + */ + public InLongMsgBinlog ignoreErrors() { + internalProperties.putBoolean(FORMAT_IGNORE_ERRORS, true); + return this; + } + + /** + * Defines the format for the next field. + * + * @param fieldName The name of the field. + * @param fieldFormatInfo The format of the field. + */ + public InLongMsgBinlog field(String fieldName, FormatInfo fieldFormatInfo) { + checkNotNull(fieldName); + checkNotNull(fieldFormatInfo); + + fieldNames.add(fieldName); + fieldFormatInfos.add(fieldFormatInfo); + + return this; + } + + /** + * Sets the format schema. Required if schema is not derived. + * + *

Note: This schema defined by this method will be removed if + * {@link #field(String, FormatInfo)} is called.

+ * + * @param rowFormatInfo The format. + */ + public InLongMsgBinlog schema(RowFormatInfo rowFormatInfo) { + checkNotNull(rowFormatInfo); + internalProperties.putString(FORMAT_SCHEMA, FormatUtils.marshall(rowFormatInfo)); + return this; + } + + /** + * Sets the format schema. Required if schema is not derived. + * + * @param schema format schema string. + */ + @Deprecated + public InLongMsgBinlog schema(String schema) { + checkNotNull(schema); + internalProperties.putString(FORMAT_SCHEMA, schema); + return this; + } + + /** + * Sets the name of the time field. + * + * @param timeFieldName The name of the time field. + */ + public InLongMsgBinlog timeFieldName(String timeFieldName) { + checkNotNull(timeFieldName); + internalProperties.putString(FORMAT_TIME_FIELD_NAME, timeFieldName); + return this; + } + + /** + * Sets the name of the attributes field. + * + * @param attributesFieldName The name of the attributes field. + */ + public InLongMsgBinlog attributesFieldName(String attributesFieldName) { + checkNotNull(attributesFieldName); + internalProperties.putString(FORMAT_ATTRIBUTES_FIELD_NAME, attributesFieldName); + return this; + } + + /** + * Sets the name of the metadata field. + * + * @param metadataFieldName The name of the metadata field. + */ + public InLongMsgBinlog metadataFieldName(String metadataFieldName) { + checkNotNull(metadataFieldName); + internalProperties.putString(FORMAT_METADATA_FIELD_NAME, metadataFieldName); + return this; + } + + @Override + protected Map toFormatProperties() { + if (!fieldNames.isEmpty()) { + RowFormatInfo rowFormatInfo = + new RowFormatInfo( + fieldNames.toArray(new String[0]), + fieldFormatInfos.toArray(new FormatInfo[0])); + + String schema = FormatUtils.marshall(rowFormatInfo); + internalProperties.putString(FORMAT_SCHEMA, schema); + } + + return internalProperties.asMap(); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatBuilder.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatBuilder.java new file mode 100644 index 00000000000..9dad5d9349e --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatBuilder.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.common.RowFormatInfo; + +import org.apache.flink.table.descriptors.DescriptorProperties; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.DEFAULT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.DEFAULT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.DEFAULT_INCLUDE_UPDATE_BEFORE; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.DEFAULT_METADATA_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.FORMAT_INCLUDE_UPDATE_BEFORE; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.FORMAT_METADATA_FIELD_NAME; + +public abstract class InLongMsgBinlogFormatBuilder { + + protected final RowFormatInfo rowFormatInfo; + + protected String timeFieldName = DEFAULT_TIME_FIELD_NAME; + protected String attributesFieldName = DEFAULT_ATTRIBUTES_FIELD_NAME; + protected String metadataFieldName = DEFAULT_METADATA_FIELD_NAME; + protected boolean ignoreErrors = DEFAULT_IGNORE_ERRORS; + protected boolean includeUpdateBefore = DEFAULT_INCLUDE_UPDATE_BEFORE; + + protected InLongMsgBinlogFormatBuilder(RowFormatInfo rowFormatInfo) { + this.rowFormatInfo = rowFormatInfo; + } + + @SuppressWarnings("unchecked") + public T setTimeFieldName(String timeFieldName) { + this.timeFieldName = timeFieldName; + return (T) this; + } + + @SuppressWarnings("unchecked") + public T setAttributesFieldName(String attributesFieldName) { + this.attributesFieldName = attributesFieldName; + return (T) this; + } + + @SuppressWarnings("unchecked") + public T setMetadataFieldName(String metadataFieldName) { + this.metadataFieldName = metadataFieldName; + return (T) this; + } + + @SuppressWarnings("unchecked") + public T setIgnoreErrors(boolean ignoreErrors) { + this.ignoreErrors = ignoreErrors; + return (T) this; + } + + @SuppressWarnings("unchecked") + public T setIncludeUpdateBefore(boolean includeUpdateBefore) { + this.includeUpdateBefore = includeUpdateBefore; + return (T) this; + } + + @SuppressWarnings("unchecked") + public T configure(DescriptorProperties descriptorProperties) { + descriptorProperties.getOptionalString(FORMAT_TIME_FIELD_NAME) + .ifPresent(this::setTimeFieldName); + descriptorProperties.getOptionalString(FORMAT_ATTRIBUTES_FIELD_NAME) + .ifPresent(this::setAttributesFieldName); + descriptorProperties.getOptionalString(FORMAT_METADATA_FIELD_NAME) + .ifPresent(this::setMetadataFieldName); + descriptorProperties.getOptionalBoolean(FORMAT_IGNORE_ERRORS) + .ifPresent(this::setIgnoreErrors); + descriptorProperties.getOptionalBoolean(FORMAT_INCLUDE_UPDATE_BEFORE) + .ifPresent(this::setIncludeUpdateBefore); + + return (T) this; + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatDeserializer.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatDeserializer.java new file mode 100644 index 00000000000..538e02e8154 --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatDeserializer.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.common.RowFormatInfo; +import org.apache.inlong.sort.formats.inlongmsg.AbstractInLongMsgFormatDeserializer; +import org.apache.inlong.sort.formats.inlongmsg.FailureHandler; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgBody; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgHead; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.types.Row; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * The deserializer for the records in TDMsgDBSync format. + */ +public final class InLongMsgBinlogFormatDeserializer extends AbstractInLongMsgFormatDeserializer { + + private static final long serialVersionUID = 1L; + + /** + * The type information of the data fields. + */ + @Nonnull + private final RowFormatInfo rowFormatInfo; + + /** + * The name of the time field. + */ + @Nullable + private final String timeFieldName; + + /** + * The name of the attributes field. + */ + @Nullable + private final String attributesFieldName; + + /** + * The name of the metadata field. + */ + @Nullable + private final String metadataFieldName; + + /** + * Controlling whether beforeColumnsList needs to be included when deserializing dbsync update data. + */ + private final boolean includeUpdateBefore; + + public InLongMsgBinlogFormatDeserializer( + @Nonnull RowFormatInfo rowFormatInfo, + @Nullable String timeFieldName, + @Nullable String attributesFieldName, + @Nullable String metadataFieldName, + boolean ignoreErrors) { + this( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + InLongMsgUtils.getDefaultExceptionHandler(ignoreErrors)); + } + + public InLongMsgBinlogFormatDeserializer( + @Nonnull RowFormatInfo rowFormatInfo, + @Nullable String timeFieldName, + @Nullable String attributesFieldName, + @Nullable String metadataFieldName, + @Nonnull FailureHandler failureHandler) { + this( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + failureHandler, + false); + } + + public InLongMsgBinlogFormatDeserializer( + @Nonnull RowFormatInfo rowFormatInfo, + @Nullable String timeFieldName, + @Nullable String attributesFieldName, + @Nullable String metadataFieldName, + boolean ignoreErrors, + boolean includeUpdateBefore) { + this( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + InLongMsgUtils.getDefaultExceptionHandler(ignoreErrors), + includeUpdateBefore); + } + + public InLongMsgBinlogFormatDeserializer( + @Nonnull RowFormatInfo rowFormatInfo, + @Nullable String timeFieldName, + @Nullable String attributesFieldName, + @Nullable String metadataFieldName, + @Nonnull FailureHandler failureHandler, + boolean includeUpdateBefore) { + super(failureHandler); + + this.rowFormatInfo = rowFormatInfo; + this.timeFieldName = timeFieldName; + this.attributesFieldName = attributesFieldName; + this.metadataFieldName = metadataFieldName; + this.includeUpdateBefore = includeUpdateBefore; + } + + @Override + protected InLongMsgHead parseHead(String attr) { + return InLongMsgBinlogUtils.parseHead(attr); + } + + @Override + protected List parseBodyList(byte[] bytes) { + return Collections.singletonList(InLongMsgBinlogUtils.parseBody(bytes)); + } + + @Override + public TypeInformation getProducedType() { + return InLongMsgBinlogUtils.getRowType( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName); + } + + @Override + protected List convertRows(InLongMsgHead head, InLongMsgBody body) throws IOException { + return InLongMsgBinlogUtils.getRows( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + head.getAttributes(), + body.getData(), + includeUpdateBefore); + } + + /** + * The builder for {@link InLongMsgBinlogFormatDeserializer}. + */ + public static class Builder extends InLongMsgBinlogFormatBuilder { + + public Builder(RowFormatInfo rowFormatInfo) { + super(rowFormatInfo); + } + + public InLongMsgBinlogFormatDeserializer build() { + return new InLongMsgBinlogFormatDeserializer( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + ignoreErrors, + includeUpdateBefore); + } + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + + if (!super.equals(object)) { + return false; + } + + InLongMsgBinlogFormatDeserializer that = (InLongMsgBinlogFormatDeserializer) object; + return rowFormatInfo.equals(that.rowFormatInfo) && + Objects.equals(timeFieldName, that.timeFieldName) && + Objects.equals(attributesFieldName, that.attributesFieldName) && + Objects.equals(metadataFieldName, that.metadataFieldName) && + includeUpdateBefore == that.includeUpdateBefore; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), rowFormatInfo, timeFieldName, attributesFieldName, + metadataFieldName, includeUpdateBefore); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatFactory.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatFactory.java new file mode 100644 index 00000000000..1f35f2a3a44 --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatFactory.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.base.TableFormatDeserializer; +import org.apache.inlong.sort.formats.base.TableFormatDeserializerFactory; +import org.apache.inlong.sort.formats.common.RowFormatInfo; +import org.apache.inlong.sort.formats.inlongmsg.AbstractInLongMsgFormatDeserializer; +import org.apache.inlong.sort.formats.inlongmsg.AbstractInLongMsgMixedFormatConverter; +import org.apache.inlong.sort.formats.inlongmsg.AbstractInLongMsgMixedFormatDeserializer; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgMixedFormatDeserializerValidator; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgMixedFormatFactory; + +import org.apache.flink.table.descriptors.DescriptorProperties; +import org.apache.flink.table.factories.TableFormatFactoryBase; +import org.apache.flink.types.Row; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_SCHEMA; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.FORMAT_INCLUDE_UPDATE_BEFORE; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.FORMAT_METADATA_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.getDataRowFormatInfo; + +/** + * Table format factory for providing configured instances of InLongMsgBinlog-to-row + * serializer and deserializer. + */ + +public final class InLongMsgBinlogFormatFactory extends TableFormatFactoryBase + implements + TableFormatDeserializerFactory, + InLongMsgMixedFormatFactory { + + public InLongMsgBinlogFormatFactory() { + super(InLongMsgBinlog.FORMAT_TYPE_VALUE, 1, false); + } + + @Override + public List supportedFormatProperties() { + final List properties = new ArrayList<>(); + properties.add(FORMAT_SCHEMA); + properties.add(FORMAT_TIME_FIELD_NAME); + properties.add(FORMAT_ATTRIBUTES_FIELD_NAME); + properties.add(FORMAT_METADATA_FIELD_NAME); + properties.add(FORMAT_IGNORE_ERRORS); + properties.add(FORMAT_INCLUDE_UPDATE_BEFORE); + + return properties; + } + + @Override + public InLongMsgBinlogFormatDeserializer createFormatDeserializer( + Map properties) { + final DescriptorProperties descriptorProperties = + new DescriptorProperties(true); + descriptorProperties.putProperties(properties); + + final InLongMsgBinlogValidator validator = new InLongMsgBinlogValidator(); + validator.validate(descriptorProperties); + + RowFormatInfo dataRowFormatInfo = getDataRowFormatInfo(descriptorProperties); + InLongMsgBinlogFormatDeserializer.Builder builder = + new InLongMsgBinlogFormatDeserializer.Builder(dataRowFormatInfo); + builder.configure(descriptorProperties); + + return builder.build(); + } + + @Override + public TableFormatDeserializer createFormatDeserializer(TableFormatDeserializer.TableFormatContext context) { + TableFormatDeserializer deserializer = + createFormatDeserializer(context.getFormatProperties()); + deserializer.init(context); + return deserializer; + } + + @Override + public InLongMsgBinlogMixedFormatDeserializer createMixedFormatDeserializer( + Map properties) { + DescriptorProperties descriptorProperties = new DescriptorProperties(true); + descriptorProperties.putProperties(properties); + + Boolean ignoreErrors = + descriptorProperties + .getOptionalBoolean(FORMAT_IGNORE_ERRORS) + .orElse(DEFAULT_IGNORE_ERRORS); + + return new InLongMsgBinlogMixedFormatDeserializer(ignoreErrors); + } + + @Override + public AbstractInLongMsgMixedFormatConverter createMixedFormatConverter( + AbstractInLongMsgMixedFormatConverter.TableFormatContext context) { + return null; + } + + @Override + public AbstractInLongMsgMixedFormatDeserializer createMixedFormatDeserializer( + AbstractInLongMsgFormatDeserializer.TableFormatContext context) { + return null; + } + + @Override + public InLongMsgBinlogMixedFormatConverter createMixedFormatConverter( + Map properties) { + DescriptorProperties descriptorProperties = new DescriptorProperties(true); + descriptorProperties.putProperties(properties); + + final InLongMsgMixedFormatDeserializerValidator validator = + new InLongMsgMixedFormatDeserializerValidator(); + validator.validate(descriptorProperties); + + RowFormatInfo dataRowFormatInfo = getDataRowFormatInfo(descriptorProperties); + InLongMsgBinlogMixedFormatConverter.Builder builder = + new InLongMsgBinlogMixedFormatConverter.Builder(dataRowFormatInfo); + builder.configure(descriptorProperties); + + return builder.build(); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogMixedFormatConverter.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogMixedFormatConverter.java new file mode 100644 index 00000000000..a5d5f8c5d1e --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogMixedFormatConverter.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.common.RowFormatInfo; +import org.apache.inlong.sort.formats.inlongmsg.AbstractInLongMsgMixedFormatConverter; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.types.Row; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Converter used to deserialize a mixed row in tdmsg-dbsync format. + */ +public class InLongMsgBinlogMixedFormatConverter extends AbstractInLongMsgMixedFormatConverter { + + private static final long serialVersionUID = 1L; + + /** + * The type information of the data fields. + */ + @Nonnull + private final RowFormatInfo rowFormatInfo; + + /** + * The name of the time field. + */ + @Nullable + private final String timeFieldName; + + /** + * The name of the attributes field. + */ + @Nullable + private final String attributesFieldName; + + /** + * The name of the metadata field. + */ + @Nullable + private final String metadataFieldName; + + /** + * Controlling whether beforeColumnsList needs to be included when deserializing dbsync update data. + */ + private final boolean includeUpdateBefore; + + public InLongMsgBinlogMixedFormatConverter( + @Nonnull RowFormatInfo rowFormatInfo, + @Nullable String timeFieldName, + @Nullable String attributesFieldName, + @Nullable String metadataFieldName, + boolean ignoreErrors) { + this( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + ignoreErrors, + false); + } + + public InLongMsgBinlogMixedFormatConverter( + @Nonnull RowFormatInfo rowFormatInfo, + @Nullable String timeFieldName, + @Nullable String attributesFieldName, + @Nullable String metadataFieldName, + boolean ignoreErrors, + boolean includeUpdateBefore) { + super(ignoreErrors); + + this.rowFormatInfo = rowFormatInfo; + this.timeFieldName = timeFieldName; + this.attributesFieldName = attributesFieldName; + this.metadataFieldName = metadataFieldName; + this.includeUpdateBefore = includeUpdateBefore; + } + + @Override + public TypeInformation getProducedType() { + return InLongMsgBinlogUtils.getRowType( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName); + } + + @Override + public List convertRows( + Map attributes, + byte[] data, + String tid, + Timestamp time, + List predefinedFields, + List fields, + Map entries) throws Exception { + return InLongMsgBinlogUtils.getRows( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + attributes, + data, + includeUpdateBefore); + } + + /** + * The builder for {@link InLongMsgBinlogMixedFormatConverter}. + */ + public static class Builder extends InLongMsgBinlogFormatBuilder { + + public Builder(RowFormatInfo rowFormatInfo) { + super(rowFormatInfo); + } + + public InLongMsgBinlogMixedFormatConverter build() { + return new InLongMsgBinlogMixedFormatConverter( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + ignoreErrors, + includeUpdateBefore); + } + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + + if (object == null || getClass() != object.getClass()) { + return false; + } + + if (!super.equals(object)) { + return false; + } + + InLongMsgBinlogMixedFormatConverter that = (InLongMsgBinlogMixedFormatConverter) object; + return rowFormatInfo.equals(that.rowFormatInfo) && + Objects.equals(timeFieldName, that.timeFieldName) && + Objects.equals(attributesFieldName, that.attributesFieldName) && + Objects.equals(metadataFieldName, that.metadataFieldName) && + includeUpdateBefore == that.includeUpdateBefore; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), rowFormatInfo, timeFieldName, attributesFieldName, + metadataFieldName, includeUpdateBefore); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogMixedFormatDeserializer.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogMixedFormatDeserializer.java new file mode 100644 index 00000000000..9e09b6c0377 --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogMixedFormatDeserializer.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.inlongmsg.AbstractInLongMsgMixedFormatDeserializer; +import org.apache.inlong.sort.formats.inlongmsg.FailureHandler; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgBody; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgHead; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.types.Row; + +import javax.annotation.Nonnull; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.DEFAULT_IGNORE_ERRORS; + +public class InLongMsgBinlogMixedFormatDeserializer extends AbstractInLongMsgMixedFormatDeserializer { + + private static final long serialVersionUID = 1L; + + public InLongMsgBinlogMixedFormatDeserializer(Boolean ignoreErrors) { + this(InLongMsgUtils.getDefaultExceptionHandler(ignoreErrors)); + } + + public InLongMsgBinlogMixedFormatDeserializer() { + this(DEFAULT_IGNORE_ERRORS); + } + + public InLongMsgBinlogMixedFormatDeserializer(@Nonnull FailureHandler failureHandler) { + super(failureHandler); + } + + @Override + public TypeInformation getProducedType() { + return InLongMsgUtils.MIXED_ROW_TYPE; + } + + @Override + protected InLongMsgHead parseHead(String attr) { + return InLongMsgBinlogUtils.parseHead(attr); + } + + @Override + protected List parseBodyList(byte[] bytes) { + return Collections.singletonList(InLongMsgBinlogUtils.parseBody(bytes)); + } + + @Override + protected List convertRows(InLongMsgHead head, InLongMsgBody body) throws IOException { + Row row = InLongMsgUtils.buildMixedRow(head, body, head.getTid()); + return Collections.singletonList(row); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + if (!super.equals(o)) { + return false; + } + + InLongMsgBinlogMixedFormatDeserializer that = (InLongMsgBinlogMixedFormatDeserializer) o; + return super.equals(that); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode()); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogUtils.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogUtils.java new file mode 100644 index 00000000000..76ad558ea80 --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogUtils.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.binlog.InLongBinlog; +import org.apache.inlong.sort.formats.common.FormatInfo; +import org.apache.inlong.sort.formats.common.RowFormatInfo; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgBody; +import org.apache.inlong.sort.formats.inlongmsg.InLongMsgHead; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.descriptors.DescriptorProperties; +import org.apache.flink.types.Row; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.inlong.sort.formats.base.TableFormatUtils.deserializeBasicField; +import static org.apache.inlong.sort.formats.base.TableFormatUtils.deserializeRowFormatInfo; +import static org.apache.inlong.sort.formats.base.TableFormatUtils.getType; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.DEFAULT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.DEFAULT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.INLONGMSG_ATTR_STREAM_ID; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.INLONGMSG_ATTR_TID; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.getPredefinedFields; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.parseAttr; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.parseEpochTime; + +public class InLongMsgBinlogUtils { + + public static final String FORMAT_METADATA_FIELD_NAME = "format.metadata-field-name"; + public static final String FORMAT_INCLUDE_UPDATE_BEFORE = "format.include-update-before"; + public static final String DEFAULT_METADATA_FIELD_NAME = "dbsync_metadata"; + public static final boolean DEFAULT_INCLUDE_UPDATE_BEFORE = false; + + public static final String METADATA_OPERATION_TYPE = "dbsync_operation_type"; + public static final String METADATA_INSTANCE_NAME = "dbsync_instance_name"; + public static final String METADATA_SCHEMA_NAME = "dbsync_schema_name"; + public static final String METADATA_TABLE_NAME = "dbsync_table_name"; + public static final String METADATA_EXECUTE_TIME = "dbsync_execute_time"; + public static final String METADATA_EXECUTE_ORDER = "dbsync_execute_order"; + public static final String METADATA_TRANSFER_IP = "dbsync_transfer_ip"; + + public static final String DBSYNC_OPERATION_INERT = "I"; + public static final String DBSYNC_OPERATION_UPDATE = "U"; + public static final String DBSYNC_OPERATION_UPDATE_BEFORE = "UB"; + public static final String DBSYNC_OPERATION_DELETE = "D"; + + public static RowFormatInfo getDataRowFormatInfo(DescriptorProperties descriptorProperties) { + RowFormatInfo rowFormatInfo = deserializeRowFormatInfo(descriptorProperties); + + Set fieldNames = new HashSet<>(); + Collections.addAll(fieldNames, rowFormatInfo.getFieldNames()); + + String timeFieldName = + descriptorProperties + .getOptionalString(FORMAT_TIME_FIELD_NAME) + .orElse(DEFAULT_TIME_FIELD_NAME); + if (timeFieldName != null && fieldNames.contains(timeFieldName)) { + throw new ValidationException("The name of the time field " + timeFieldName + + " conflicts with one of the data fields."); + } + + String attributesFieldName = + descriptorProperties + .getOptionalString(FORMAT_ATTRIBUTES_FIELD_NAME) + .orElse(DEFAULT_ATTRIBUTES_FIELD_NAME); + if (attributesFieldName != null && fieldNames.contains(attributesFieldName)) { + throw new ValidationException("The name of the attributes field " + + attributesFieldName + " conflicts with one of the data fields."); + } + + String metadataFieldName = + descriptorProperties + .getOptionalString(FORMAT_METADATA_FIELD_NAME) + .orElse(DEFAULT_METADATA_FIELD_NAME); + if (metadataFieldName != null && fieldNames.contains(metadataFieldName)) { + throw new ValidationException("The name of the metadata field " + + metadataFieldName + " conflicts with one of the data fields."); + } + + return rowFormatInfo; + } + + public static InLongMsgHead parseHead(String attr) { + Map attributes = parseAttr(attr); + + // Extracts interface from the attributes. + String tid; + if (attributes.containsKey(INLONGMSG_ATTR_STREAM_ID)) { + tid = attributes.get(INLONGMSG_ATTR_STREAM_ID); + } else if (attributes.containsKey(INLONGMSG_ATTR_TID)) { + tid = attributes.get(INLONGMSG_ATTR_TID); + } else { + throw new IllegalArgumentException( + "Could not find " + INLONGMSG_ATTR_STREAM_ID + + " or " + INLONGMSG_ATTR_TID + " in attributes!"); + } + + // Extracts time from the attributes + Timestamp time = parseEpochTime(Long.toString(System.currentTimeMillis())); + List predefinedFields = getPredefinedFields(attributes); + + return new InLongMsgHead(attributes, tid, time, predefinedFields); + } + + public static InLongMsgBody parseBody(byte[] bytes) { + return new InLongMsgBody( + bytes, + null, + Collections.emptyList(), + Collections.emptyMap()); + } + + public static TypeInformation getRowType( + RowFormatInfo rowFormatInfo, + String timeFieldName, + String attributesFieldName, + String metadataFieldName) { + List fieldNames = new ArrayList<>(); + List> fieldTypes = new ArrayList<>(); + + if (timeFieldName != null) { + fieldNames.add(timeFieldName); + fieldTypes.add(Types.SQL_TIMESTAMP); + } + + if (attributesFieldName != null) { + fieldNames.add(attributesFieldName); + fieldTypes.add(Types.MAP(Types.STRING, Types.STRING)); + } + + if (metadataFieldName != null) { + fieldNames.add(metadataFieldName); + fieldTypes.add(Types.MAP(Types.STRING, Types.STRING)); + } + + String[] dataFieldNames = rowFormatInfo.getFieldNames(); + FormatInfo[] dataFieldFormatInfos = rowFormatInfo.getFieldFormatInfos(); + + for (int i = 0; i < dataFieldNames.length; ++i) { + fieldNames.add(dataFieldNames[i]); + fieldTypes.add(getType(dataFieldFormatInfos[i].getTypeInfo())); + } + + return Types.ROW_NAMED( + fieldNames.toArray(new String[0]), + fieldTypes.toArray(new TypeInformation[0])); + } + + public static List getRows( + RowFormatInfo rowFormatInfo, + String timeFieldName, + String attributesFieldName, + String metadataFieldName, + Map attributes, + byte[] bytes, + boolean includeUpdateBefore) throws IOException { + + InLongBinlog.RowData rowData = InLongBinlog.RowData.parseFrom(bytes); + + List rows = new ArrayList<>(); + + switch (rowData.getEventType()) { + case INSERT: + rows.add( + constructRow( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + attributes, + rowData, + DBSYNC_OPERATION_INERT, + rowData.getAfterColumnsList())); + break; + case UPDATE: + if (includeUpdateBefore) { + rows.add( + constructRow( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + attributes, + rowData, + DBSYNC_OPERATION_UPDATE_BEFORE, + rowData.getBeforeColumnsList())); + } + rows.add( + constructRow( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + attributes, + rowData, + DBSYNC_OPERATION_UPDATE, + rowData.getAfterColumnsList())); + break; + case DELETE: + rows.add( + constructRow( + rowFormatInfo, + timeFieldName, + attributesFieldName, + metadataFieldName, + attributes, + rowData, + DBSYNC_OPERATION_DELETE, + rowData.getBeforeColumnsList())); + break; + default: + return null; + } + + return rows; + } + + private static Row constructRow( + RowFormatInfo rowFormatInfo, + String timeFieldName, + String attributesFieldName, + String metadataFieldName, + Map attributes, + InLongBinlog.RowData rowData, + String operation, + List columns) { + List headFields = new ArrayList<>(); + + if (timeFieldName != null) { + headFields.add(new Timestamp(rowData.getExecuteTime())); + } + + if (attributesFieldName != null) { + headFields.add(attributes); + } + + if (metadataFieldName != null) { + Map metadata = new HashMap<>(); + + metadata.put(METADATA_INSTANCE_NAME, rowData.getInstanceName()); + metadata.put(METADATA_SCHEMA_NAME, rowData.getSchemaName()); + metadata.put(METADATA_TABLE_NAME, rowData.getTableName()); + metadata.put(METADATA_OPERATION_TYPE, operation); + metadata.put(METADATA_EXECUTE_TIME, Long.toString(rowData.getExecuteTime())); + metadata.put(METADATA_EXECUTE_ORDER, Long.toString(rowData.getExecuteOrder())); + metadata.put(METADATA_TRANSFER_IP, rowData.getTransferIp()); + + headFields.add(metadata); + } + + String[] dataFieldNames = rowFormatInfo.getFieldNames(); + FormatInfo[] dataFieldFormatInfos = rowFormatInfo.getFieldFormatInfos(); + + Row row = new Row(dataFieldNames.length + headFields.size()); + + for (int i = 0; i < headFields.size(); ++i) { + row.setField(i, headFields.get(i)); + } + + Map columnMap = new HashMap<>(columns.size()); + for (InLongBinlog.Column column : columns) { + if (column.getIsNull()) { + columnMap.put(column.getName(), null); + } else { + columnMap.put(column.getName(), column.getValue()); + } + } + + for (int i = 0; i < dataFieldNames.length; ++i) { + String fieldName = dataFieldNames[i]; + String fieldText = columnMap.get(fieldName); + + if (fieldText == null) { + row.setField(i + headFields.size(), null); + } else { + Object field = + deserializeBasicField( + fieldName, + dataFieldFormatInfos[i], + fieldText, + null); + row.setField(i + headFields.size(), field); + } + } + + return row; + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogValidator.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogValidator.java new file mode 100644 index 00000000000..d8957ee142e --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogValidator.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.base.FormatDescriptorValidator; + +import org.apache.flink.table.descriptors.DescriptorProperties; + +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_IGNORE_ERRORS; +import static org.apache.inlong.sort.formats.base.TableFormatConstants.FORMAT_SCHEMA; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.FORMAT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.FORMAT_INCLUDE_UPDATE_BEFORE; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.FORMAT_METADATA_FIELD_NAME; + +public class InLongMsgBinlogValidator extends FormatDescriptorValidator { + + @Override + public void validate(DescriptorProperties properties) { + super.validate(properties); + + properties.validateString(FORMAT_SCHEMA, false); + properties.validateString(FORMAT_TIME_FIELD_NAME, true, 1); + properties.validateString(FORMAT_ATTRIBUTES_FIELD_NAME, true, 1); + properties.validateString(FORMAT_METADATA_FIELD_NAME, true, 1); + properties.validateBoolean(FORMAT_IGNORE_ERRORS, true); + properties.validateBoolean(FORMAT_INCLUDE_UPDATE_BEFORE, true); + } + +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory new file mode 100644 index 00000000000..af0936add1d --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogFormatFactory \ No newline at end of file diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatDeserializerTest.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatDeserializerTest.java new file mode 100644 index 00000000000..5bcca7f72ab --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatDeserializerTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.common.FormatInfo; +import org.apache.inlong.sort.formats.common.IntFormatInfo; +import org.apache.inlong.sort.formats.common.RowFormatInfo; +import org.apache.inlong.sort.formats.common.StringFormatInfo; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.types.Row; +import org.junit.Test; + +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.DEFAULT_ATTRIBUTES_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsg.InLongMsgUtils.DEFAULT_TIME_FIELD_NAME; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.DEFAULT_METADATA_FIELD_NAME; +import static org.junit.Assert.assertEquals; + +public class InLongMsgBinlogFormatDeserializerTest { + + private static final RowFormatInfo TEST_ROW_INFO = + new RowFormatInfo( + new String[]{"f1", "f2", "f3", "f4", "f5", "f6"}, + new FormatInfo[]{ + IntFormatInfo.INSTANCE, + IntFormatInfo.INSTANCE, + IntFormatInfo.INSTANCE, + StringFormatInfo.INSTANCE, + StringFormatInfo.INSTANCE, + StringFormatInfo.INSTANCE + }); + + @Test + public void testRowType() { + + InLongMsgBinlogFormatDeserializer deserializer = + new InLongMsgBinlogFormatDeserializer.Builder(TEST_ROW_INFO).build(); + + TypeInformation expectedRowType = + Types.ROW_NAMED( + new String[]{ + DEFAULT_TIME_FIELD_NAME, + DEFAULT_ATTRIBUTES_FIELD_NAME, + DEFAULT_METADATA_FIELD_NAME, + "f1", + "f2", + "f3", + "f4", + "f5", + "f6" + }, + Types.SQL_TIMESTAMP, + Types.MAP(Types.STRING, Types.STRING), + Types.MAP(Types.STRING, Types.STRING), + Types.INT, + Types.INT, + Types.INT, + Types.STRING, + Types.STRING, + Types.STRING); + + assertEquals(expectedRowType, deserializer.getProducedType()); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatFactoryTest.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatFactoryTest.java new file mode 100644 index 00000000000..35ad72d6dcb --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogFormatFactoryTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.base.TableFormatDeserializer; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; +import org.apache.inlong.sort.formats.common.DateFormatInfo; +import org.apache.inlong.sort.formats.common.FormatInfo; +import org.apache.inlong.sort.formats.common.IntFormatInfo; +import org.apache.inlong.sort.formats.common.RowFormatInfo; +import org.apache.inlong.sort.formats.common.StringFormatInfo; + +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class InLongMsgBinlogFormatFactoryTest { + + private static final RowFormatInfo TEST_FORMAT_SCHEMA = + new RowFormatInfo( + new String[]{"student_name", "score", "date"}, + new FormatInfo[]{ + StringFormatInfo.INSTANCE, + IntFormatInfo.INSTANCE, + new DateFormatInfo("yyyy-MM-dd") + }); + + @Test + public void testCreateTableFormatDeserializer() throws Exception { + final Map properties = + new InLongMsgBinlog() + .schema(TEST_FORMAT_SCHEMA) + .toProperties(); + assertNotNull(properties); + + final InLongMsgBinlogFormatDeserializer expectedDeserializer = + new InLongMsgBinlogFormatDeserializer.Builder(TEST_FORMAT_SCHEMA) + .build(); + + final TableFormatDeserializer actualDeserializer = + TableFormatForRowUtils.getTableFormatDeserializer( + properties, + getClass().getClassLoader()); + + assertEquals(expectedDeserializer, actualDeserializer); + } +} \ No newline at end of file diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogTest.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogTest.java new file mode 100644 index 00000000000..7b970d3a20a --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/java/org/apache/inlong/sort/formats/inlongmsgbinlog/InLongMsgBinlogTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.formats.inlongmsgbinlog; + +import org.apache.inlong.sort.formats.common.DateFormatInfo; +import org.apache.inlong.sort.formats.common.FormatInfo; +import org.apache.inlong.sort.formats.common.IntFormatInfo; +import org.apache.inlong.sort.formats.common.RowFormatInfo; +import org.apache.inlong.sort.formats.common.StringFormatInfo; + +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.descriptors.Descriptor; +import org.apache.flink.table.descriptors.DescriptorTestBase; +import org.apache.flink.table.descriptors.DescriptorValidator; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.inlong.sort.formats.common.FormatUtils.marshall; +import static org.apache.inlong.sort.formats.inlongmsgbinlog.InLongMsgBinlogUtils.DEFAULT_METADATA_FIELD_NAME; + +public class InLongMsgBinlogTest extends DescriptorTestBase { + + private static final RowFormatInfo TEST_SCHEMA = + new RowFormatInfo( + new String[]{"student_name", "score", "date"}, + new FormatInfo[]{ + StringFormatInfo.INSTANCE, + IntFormatInfo.INSTANCE, + new DateFormatInfo("yyyy-MM-dd") + }); + + private static final Descriptor CUSTOM_DESCRIPTOR_WITH_SCHEMA = + new InLongMsgBinlog() + .schema(TEST_SCHEMA) + .timeFieldName("time") + .attributesFieldName("attributes") + .metadataFieldName(DEFAULT_METADATA_FIELD_NAME) + .ignoreErrors(); + + @Test(expected = ValidationException.class) + public void testInvalidIgnoreParseErrors() { + addPropertyAndVerify(CUSTOM_DESCRIPTOR_WITH_SCHEMA, "format.ignore-errors", "DDD"); + } + + @Test(expected = ValidationException.class) + public void testMissingSchema() { + removePropertyAndVerify(CUSTOM_DESCRIPTOR_WITH_SCHEMA, "format.schema"); + } + + // -------------------------------------------------------------------------------------------- + + @Override + public List descriptors() { + return Collections.singletonList(CUSTOM_DESCRIPTOR_WITH_SCHEMA); + } + + @Override + public List> properties() { + final Map props1 = new HashMap<>(); + props1.put("format.type", "InLongMsg-Binlog"); + props1.put("format.property-version", "1"); + props1.put("format.schema", marshall(TEST_SCHEMA)); + props1.put("format.time-field-name", "time"); + props1.put("format.attributes-field-name", "attributes"); + props1.put("format.metadata-field-name", DEFAULT_METADATA_FIELD_NAME); + props1.put("format.ignore-errors", "true"); + + return Collections.singletonList(props1); + } + + @Override + public DescriptorValidator validator() { + return new InLongMsgBinlogValidator(); + } +} diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/resources/log4j-test.properties b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/resources/log4j-test.properties new file mode 100644 index 00000000000..d35bf39669d --- /dev/null +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-binlog/src/test/resources/log4j-test.properties @@ -0,0 +1,17 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ \ No newline at end of file diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-csv/src/test/java/org/apache/inlong/sort/formats/inlongmsgcsv/InLongMsgCsvFormatFactoryTest.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-csv/src/test/java/org/apache/inlong/sort/formats/inlongmsgcsv/InLongMsgCsvFormatFactoryTest.java index 2ffeab9e9b1..8adaa6acbf7 100644 --- a/inlong-sort/sort-formats/format-row/format-inlongmsg-csv/src/test/java/org/apache/inlong/sort/formats/inlongmsgcsv/InLongMsgCsvFormatFactoryTest.java +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-csv/src/test/java/org/apache/inlong/sort/formats/inlongmsgcsv/InLongMsgCsvFormatFactoryTest.java @@ -18,7 +18,7 @@ package org.apache.inlong.sort.formats.inlongmsgcsv; import org.apache.inlong.sort.formats.base.TableFormatDeserializer; -import org.apache.inlong.sort.formats.base.TableFormatUtils; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.inlong.sort.formats.common.DateFormatInfo; import org.apache.inlong.sort.formats.common.FormatInfo; import org.apache.inlong.sort.formats.common.FormatUtils; @@ -95,7 +95,7 @@ public void testCreateTableFormatDeserializer() throws Exception { false); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); @@ -116,7 +116,7 @@ public void testCreateTableFormatDeserializerWithDerivation() { new InLongMsgCsvFormatDeserializer.Builder(TEST_FORMAT_SCHEMA).build(); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); diff --git a/inlong-sort/sort-formats/format-row/format-inlongmsg-kv/src/test/java/org/apache/inlong/sort/formats/inlongmsgkv/InLongMsgKvFormatFactoryTest.java b/inlong-sort/sort-formats/format-row/format-inlongmsg-kv/src/test/java/org/apache/inlong/sort/formats/inlongmsgkv/InLongMsgKvFormatFactoryTest.java index 54352acb3cb..659b01ce62d 100644 --- a/inlong-sort/sort-formats/format-row/format-inlongmsg-kv/src/test/java/org/apache/inlong/sort/formats/inlongmsgkv/InLongMsgKvFormatFactoryTest.java +++ b/inlong-sort/sort-formats/format-row/format-inlongmsg-kv/src/test/java/org/apache/inlong/sort/formats/inlongmsgkv/InLongMsgKvFormatFactoryTest.java @@ -18,7 +18,7 @@ package org.apache.inlong.sort.formats.inlongmsgkv; import org.apache.inlong.sort.formats.base.TableFormatDeserializer; -import org.apache.inlong.sort.formats.base.TableFormatUtils; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.inlong.sort.formats.common.DateFormatInfo; import org.apache.inlong.sort.formats.common.FormatInfo; import org.apache.inlong.sort.formats.common.IntFormatInfo; @@ -94,7 +94,7 @@ public void testCreateTableFormatDeserializer() throws Exception { false); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); @@ -114,7 +114,7 @@ public void testCreateTableFormatDeserializerWithDerivation() { new InLongMsgKvFormatDeserializer.Builder(TEST_FORMAT_SCHEMA).build(); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); diff --git a/inlong-sort/sort-formats/format-row/format-kv/src/main/java/org/apache/inlong/sort/formats/kv/KvDeserializationSchema.java b/inlong-sort/sort-formats/format-row/format-kv/src/main/java/org/apache/inlong/sort/formats/kv/KvDeserializationSchema.java index 193e35e7af2..c5de08cae63 100644 --- a/inlong-sort/sort-formats/format-row/format-kv/src/main/java/org/apache/inlong/sort/formats/kv/KvDeserializationSchema.java +++ b/inlong-sort/sort-formats/format-row/format-kv/src/main/java/org/apache/inlong/sort/formats/kv/KvDeserializationSchema.java @@ -18,7 +18,7 @@ package org.apache.inlong.sort.formats.kv; import org.apache.inlong.sort.formats.base.DefaultDeserializationSchema; -import org.apache.inlong.sort.formats.base.TableFormatUtils; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.inlong.sort.formats.common.FormatInfo; import org.apache.inlong.sort.formats.common.RowFormatInfo; @@ -128,7 +128,7 @@ public KvDeserializationSchema( @SuppressWarnings("unchecked") @Override public TypeInformation getProducedType() { - return (TypeInformation) TableFormatUtils.getType(rowFormatInfo.getTypeInfo()); + return (TypeInformation) TableFormatForRowUtils.getType(rowFormatInfo.getTypeInfo()); } @Override diff --git a/inlong-sort/sort-formats/format-row/format-kv/src/test/java/org/apache/inlong/sort/formats/kv/KvFormatFactoryTest.java b/inlong-sort/sort-formats/format-row/format-kv/src/test/java/org/apache/inlong/sort/formats/kv/KvFormatFactoryTest.java index 939feb60d0f..021cb89fa38 100644 --- a/inlong-sort/sort-formats/format-row/format-kv/src/test/java/org/apache/inlong/sort/formats/kv/KvFormatFactoryTest.java +++ b/inlong-sort/sort-formats/format-row/format-kv/src/test/java/org/apache/inlong/sort/formats/kv/KvFormatFactoryTest.java @@ -20,8 +20,8 @@ import org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer; import org.apache.inlong.sort.formats.base.DefaultTableFormatSerializer; import org.apache.inlong.sort.formats.base.TableFormatDeserializer; +import org.apache.inlong.sort.formats.base.TableFormatForRowUtils; import org.apache.inlong.sort.formats.base.TableFormatSerializer; -import org.apache.inlong.sort.formats.base.TableFormatUtils; import org.apache.inlong.sort.formats.common.DateFormatInfo; import org.apache.inlong.sort.formats.common.FormatInfo; import org.apache.inlong.sort.formats.common.IntFormatInfo; @@ -109,7 +109,7 @@ public void testCreateTableFormatDeserializer() throws Exception { new DefaultTableFormatDeserializer(deserializationSchema); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); @@ -132,7 +132,7 @@ public void testCreateTableFormatDeserializerWithDerivation() { new DefaultTableFormatDeserializer(deserializationSchema); final TableFormatDeserializer actualDeser = - TableFormatUtils.getTableFormatDeserializer( + TableFormatForRowUtils.getTableFormatDeserializer( properties, getClass().getClassLoader()); @@ -167,7 +167,7 @@ public void testCreateTableFormatSerializer() throws Exception { new DefaultTableFormatSerializer(serializationSchema); final TableFormatSerializer actualSer = - TableFormatUtils.getTableFormatSerializer( + TableFormatForRowUtils.getTableFormatSerializer( properties, getClass().getClassLoader()); @@ -190,7 +190,7 @@ public void testCreateFormatSerializerDerivation() { new DefaultTableFormatSerializer(serializationSchema); final TableFormatSerializer actualSer = - TableFormatUtils.getTableFormatSerializer( + TableFormatForRowUtils.getTableFormatSerializer( properties, getClass().getClassLoader()); @@ -223,7 +223,7 @@ public void testCreateProjectedDeserializationSchema() throws IOException { false); final DeserializationSchema actualDeser = - TableFormatUtils.getProjectedDeserializationSchema( + TableFormatForRowUtils.getProjectedDeserializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); @@ -244,7 +244,7 @@ public void testCreateProjectedDeserializationSchemaWithDerivation() { new KvDeserializationSchema(TEST_FORMAT_SCHEMA); final DeserializationSchema actualDeser = - TableFormatUtils.getProjectedDeserializationSchema( + TableFormatForRowUtils.getProjectedDeserializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); @@ -278,7 +278,7 @@ public void testCreateProjectedSerializationSchema() throws IOException { false); final SerializationSchema actualSer = - TableFormatUtils.getProjectedSerializationSchema( + TableFormatForRowUtils.getProjectedSerializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); @@ -299,7 +299,7 @@ public void testCreateProjectedSerializationSchemaWithDerivation() { new KvSerializationSchema(TEST_FORMAT_SCHEMA); final SerializationSchema actualSer = - TableFormatUtils.getProjectedSerializationSchema( + TableFormatForRowUtils.getProjectedSerializationSchema( properties, new int[]{1, 2, 3}, getClass().getClassLoader()); diff --git a/inlong-sort/sort-formats/format-row/pom.xml b/inlong-sort/sort-formats/format-row/pom.xml index 2e36aa97771..4b74124162f 100644 --- a/inlong-sort/sort-formats/format-row/pom.xml +++ b/inlong-sort/sort-formats/format-row/pom.xml @@ -36,6 +36,7 @@ format-csv format-kv format-inlongmsg-base + format-inlongmsg-binlog format-inlongmsg-csv format-inlongmsg-kv format-inlongmsg-pb diff --git a/inlong-sort/sort-formats/pom.xml b/inlong-sort/sort-formats/pom.xml index 57cd3bca7ba..3b45e20e0c3 100644 --- a/inlong-sort/sort-formats/pom.xml +++ b/inlong-sort/sort-formats/pom.xml @@ -41,6 +41,8 @@ 1C true log4j2-test.properties + 1.7.1 + 0.6.1