support avro output for kafka

Cargo.toml

@@ -40,11 +40,10 @@ bytes = "1.0"
 configparser = "3.0.2"
 async-mutex = "1.4.0"
 project-root = "0.2.2"
-strum = { version = "0.24.1", features = ["derive"] }
+strum = { version = "0.25.0", features = ["derive"] }
 regex = "1.5.4"
 rdkafka = {version = "0.34.0", features = ["libz-static"] }
-# rdkafka = "0.34.0"
-kafka = "0.9.0"
+kafka = "0.10.0"
 reqwest = "0.11.16"
 rusoto_core = "0.48.0"
 rusoto_sqs = "0.48.0"
@@ -56,4 +55,5 @@ mongodb = { version = "2.5.0" }
 dotenv = "0.15.0"
 redis =  { version = "0.23.1", features = ["tokio-comp"] } 
 thiserror = "1.0.44"
-async-std = "1.12.0"
+async-std = "1.12.0"
+apache-avro =  { version = "0.16.0", features = ["derive"]}

Dockerfile

@@ -1,12 +1,9 @@
 # ARG DIST_IMG=gcr.io/distroless/cc:nonroot
-# ARG DIST_IMG=gcr.io/distroless/cc:debug
-ARG DIST_IMG=gcr.io/distroless/cc-debian12:debug
-# ARG DIST_IMG=debian:buster-slim
+ARG DIST_IMG=gcr.io/distroless/cc:debug
 
-ARG RUST_VERSION=1.70.0
+ARG RUST_VERSION=1.69.0
 # ARG RUST_IMG_ALT=-slim-bullseye
-# ARG RUST_IMG_ALT=-bullseye
-ARG RUST_IMG_ALT=-bookworm
+ARG RUST_IMG_ALT=-bullseye
 
 FROM --platform=${BUILDPLATFORM} rust:${RUST_VERSION}${RUST_IMG_ALT} as builder
 
@@ -36,8 +33,7 @@ RUN --mount=type=cache,target=$CARGO_HOME/git,rw \
 #     apt install --no-install-recommends -y \
 #         pkg-config \
 #         libssl-dev \
-#         libcrypto++-dev \
-#         zlib1g
+#         libcrypto++-dev
 
 WORKDIR /app
 COPY . ./
@@ -46,6 +42,13 @@ COPY . ./
 #     --mount=type=cache,target=$CARGO_HOME/registry,rw \
 #     cargo update
 
+# RUN --mount=type=cache,target=$CARGO_HOME/git,rw \
+#     --mount=type=cache,target=$CARGO_HOME/registry,rw \
+#     --mount=type=cache,target=/app/target,rw \
+#     cargo build --release ${BUILD_ARGS} && \
+#     mkdir -p bin/ && \
+#     cp /app/target/release/${MODULE_NAME} bin/
+
 RUN --mount=type=cache,target=$CARGO_HOME/git,rw \
     --mount=type=cache,target=$CARGO_HOME/registry,rw \
     --mount=type=cache,target=/app/target,rw \

dt-common/Cargo.toml

@@ -24,4 +24,5 @@ regex = { workspace = true }
 nom = { workspace = true }
 tokio = { workspace = true }
 thiserror = { workspace = true }
-kafka = { workspace = true }
+kafka = { workspace = true }
+apache-avro = {workspace = true}

dt-common/src/error.rs

@@ -46,4 +46,10 @@ pub enum Error {
 
     #[error("kafka error: {0}")]
     KafkaError(#[from] kafka::Error),
+
+    #[error("avro encode error: {0}")]
+    AvroEncodeError(#[from] apache_avro::Error),
+
+    #[error("enum parse error: {0}")]
+    EnumParseError(#[from] strum::ParseError),
 }

dt-connector/Cargo.toml

@@ -39,4 +39,5 @@ log4rs = { workspace = true }
 redis = { workspace = true }
 thiserror = { workspace = true }
 async-std = { workspace = true }
-chrono =  { workspace = true }
+chrono =  { workspace = true }
+apache-avro = {workspace = true}