Merge Release/2024 q2 into master

.devcontainer/Dockerfile

@@ -1,28 +1,30 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.217.4/containers/java/.devcontainer/base.Dockerfile
+# Install openJDK version 21 (includes maven, gradle, and node)
+FROM cimg/openjdk:21.0.2-node
 
-# [Choice] Java version (use -bullseye variants on local arm64/Apple Silicon): 11, 17, 11-bullseye, 17-bullseye, 11-buster, 17-buster
-ARG VARIANT="17"
-FROM mcr.microsoft.com/vscode/devcontainers/java:0-${VARIANT}
+# set user to root to allow apt-get to run
+USER root
 
-# [Option] Install Maven
-ARG INSTALL_MAVEN="false"
-ARG MAVEN_VERSION=""
-# [Option] Install Gradle
-ARG INSTALL_GRADLE="false"
-ARG GRADLE_VERSION=""
-RUN if [ "${INSTALL_MAVEN}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/sdkman/bin/sdkman-init.sh && sdk install maven \"${MAVEN_VERSION}\""; fi \
- && if [ "${INSTALL_GRADLE}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/sdkman/bin/sdkman-init.sh && sdk install gradle \"${GRADLE_VERSION}\""; fi
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
 
-# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
-ARG NODE_VERSION="none"
-RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
-
-# [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-# && apt-get -y install --no-install-recommends <your-package-list-here>
+# Create non-root user vscode with sudo support
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+ #
+ # Create a non-root user to use if preferred - see https://aka.ms/vscode-remote/containers/non-root-user.
+ && groupadd --gid $USER_GID $USERNAME \
+ && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \
+ && apt-get install -y sudo \
+ && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME\
+ && chmod 0440 /etc/sudoers.d/$USERNAME
 
 # [Optional] Uncomment this line to install global node packages.
-# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
+# RUN npm install -g <your-package-list-here>
 
 # install kafkacat for testing purposes
-RUN apt-get update && apt-get install -y kafkacat
+RUN apt-get update && apt-get install -y kafkacat
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+# && apt-get -y install --no-install-recommends <your-package-list-here>

.devcontainer/devcontainer.json

@@ -3,29 +3,18 @@
 {
  "name": "Java",
  "build": {
- "dockerfile": "Dockerfile",
- "args": {
- // Update the VARIANT arg to pick a Java version: 11, 17
- // Append -bullseye or -buster to pin to an OS version.
- // Use the -bullseye variants on local arm64/Apple Silicon.
- "VARIANT": "11",
- // Options
- "INSTALL_MAVEN": "true",
- "INSTALL_GRADLE": "false",
- "NODE_VERSION": "none"
- }
- },
-
- // Set *default* container specific settings.json values on container create.
- "settings": { 
- "java.home": "/docker-java-home"
+ "dockerfile": "Dockerfile"
  },
 
  // Add the IDs of extensions you want installed when the container is created.
  "extensions": [
  "vscjava.vscode-java-pack"
  ],
 
+ "containerEnv": {
+ "SHELL": "/bin/bash"
+ },
+
  // Use 'forwardPorts' to make a list of ports inside the container available locally.
  // "forwardPorts": [],
 

.github/workflows/dockerhub.yml

@@ -20,8 +20,12 @@ jobs:
  with:
  username: ${{ secrets.DOCKERHUB_USERNAME }}
  password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - name: Replcae Docker tag
+ id: set_tag
+ run: echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g')" >> $GITHUB_ENV
+
  - name: Build
  uses: docker/build-push-action@v3
  with:
  push: true
- tags: usdotjpoode/jpo-s3-deposit:${{ github.ref_name }}
+ tags: usdotjpoode/jpo-s3-deposit:${{ env.TAG }} 

.vscode/launch.json

@@ -6,40 +6,11 @@
  "configurations": [
  {
  "type": "java",
- "name": "S3D Local",
+ "name": "S3D",
  "request": "launch",
  "mainClass": "us.dot.its.jpo.ode.aws.depositor.AwsDepositor",
  "projectName": "jpo-aws-depositor",
- "args": ["--bootstrap-server", "<kafka host>:9092",
- "-d", "s3",
- "-g", "testgroup",
- "-k", "testkey",
- "-b", "s3dtestbucket",
- "-t", "test"],
- "env": {
- "AWS_ACCESS_KEY_ID": "<access key id>",
- "AWS_SECRET_KEY": "<access secret key>"
- }
- },
- {
- "type": "java",
- "name": "S3D CC",
- "request": "launch",
- "mainClass": "us.dot.its.jpo.ode.aws.depositor.AwsDepositor",
- "projectName": "jpo-aws-depositor",
- "args": ["--bootstrap-server", "<kafka host host>:9092",
- "-g", "testgroup",
- "-t", "topic.OdeBsmJson",
- "-b", "s3dtestbucket",
- "-k", "testkey",
- "-d", "s3"],
- "env": {
- "AWS_ACCESS_KEY_ID": "<access key id>",
- "AWS_SECRET_KEY": "<access secret key>",
- "KAFKA_TYPE": "CONFLUENT",
- "CONFLUENT_KEY": "<confluent key>",
- "CONFLUENT_SECRET": "<confluent secret>"
- }
+ "envFile": "${workspaceFolder}/.env",
  }
  ]
 }

README.md

@@ -1,10 +1,13 @@
-# AWS Deposit Service
+# Message Deposit Service
 
-This project is intended to serve as a  consumer application to subscribe to a Kafka topic of streaming JSON, package the results as a JSON file, and deposits the resulting file into a predetermined Firehose/Kinesis or S3 bucket. This runs alongside the ODE and when deployed using Docker Compose, runs in a Docker container.
+This project is intended to serve as a consumer application to subscribe to a Kafka topic of streaming JSON, package the results as a JSON file, and deposit the resulting file into a predetermined Firehose/Kinesis, S3 Bucket, or Google Cloud Storage Bucket (GCS). This runs alongside the ODE and when deployed using Docker Compose, runs in a Docker container.
 
 ## Quick Run
 The use of AWS credentials is being read from the machine's environmental variables. You may also set them in your bash profile. Note that when using Docker Compose from the main `jpo-ode` repository, these variables are set in the `.env` present in that repo.
 
+If depositing to GCS, credentials are read from a JSON service account key file. A sample service account file can be found at ./resources/google/sample_gcp_service_account.json. 
+Please note that if depositing to GCS the service account will need the storage.buckets.get and storage.objects.create permissions.
+
 ```
 export K_AWS_ACCESS_KEY_ID = AccessKeyId
 export K_AWS_SECRET_ACCESS_SECRET = SecretAccessKey

docker-compose-confluent-cloud.yml

@@ -22,4 +22,10 @@ services:
  KAFKA_TYPE: ${KAFKA_TYPE}
  CONFLUENT_KEY: ${CONFLUENT_KEY}
  CONFLUENT_SECRET: ${CONFLUENT_SECRET}
+ KAFKA_ENABLE_AUTO_COMMIT: ${KAFKA_ENABLE_AUTO_COMMIT}
+ KAFKA_AUTO_COMMIT_INTERVAL_MS: ${KAFKA_AUTO_COMMIT_INTERVAL_MS}
+ KAFKA_SESSION_TIMEOUT_MS: ${KAFKA_SESSION_TIMEOUT_MS}
+ GOOGLE_APPLICATION_CREDENTIALS: '/google/gcp_service_account.json'
+ volumes:
+ - ${GOOGLE_APPLICATION_CREDENTIALS}:/google/gcp_service_account.json
 

docker-compose.yml

@@ -35,6 +35,12 @@ services:
  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
  HEADER_ACCEPT: ${HEADER_ACCEPT}
  HEADER_X_API_KEY: ${HEADER_X_API_KEY}
+ KAFKA_ENABLE_AUTO_COMMIT: ${KAFKA_ENABLE_AUTO_COMMIT}
+ KAFKA_AUTO_COMMIT_INTERVAL_MS: ${KAFKA_AUTO_COMMIT_INTERVAL_MS}
+ KAFKA_SESSION_TIMEOUT_MS: ${KAFKA_SESSION_TIMEOUT_MS}
+ GOOGLE_APPLICATION_CREDENTIALS: '/google/gcp_service_account.json'
+ volumes:
+ - ${GOOGLE_APPLICATION_CREDENTIALS}:/google/gcp_service_account.json
  depends_on:
  - kafka
 

docs/Release_notes.md

@@ -1,6 +1,20 @@
 Jpo-s3-deposit Release Notes
 ----------------------------
 
+Version 1.5.0, released June 2024
+----------------------------------------
+### **Summary**
+The changes for the jpo-s3-deposit 1.5.0 release include updated hard-coded Kafka properties to be configurable, added support for GCS BLOB storage & an updated Java version for the dev container.
+
+Enhancements in this release
+- CDOT PR 18: Updated hard-coded Kafka properties to be configurable
+- CDOT PR 19: Added support for GCS BLOB storage
+- CDOT PR 22: Updated dev container Java version to 21
+
+Known Issues:
+- No known issues at this time.
+
+
 Version 1.4.0, released February 2024
 ----------------------------------------
 

pom.xml

@@ -4,7 +4,7 @@
 
  <groupId>usdot.jpo.ode</groupId>
  <artifactId>jpo-aws-depositor</artifactId>
- <version>1.4.0-SNAPSHOT</version>
+ <version>1.5.0-SNAPSHOT</version>
  <packaging>jar</packaging>
  <name>JPO AWS Depositor</name>
  <properties>
@@ -73,6 +73,12 @@
  <version>1.7.36</version>
  <scope>compile</scope>
  </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ <version>2.37.0</version>
+ </dependency> 
  </dependencies>
  <build>
  <finalName>${project.artifactId}</finalName>

resources/google/.gitignore

@@ -0,0 +1,2 @@
+*.json
+!sample_gcp_service_account.json

resources/google/README.md

@@ -0,0 +1,3 @@
+# Google Resources
+
+This folder is used as a location to put the GCP service account json file that will be used for the S3 Depositor. Please name the file gcp_service_account.json, a sample file has been provided in this folder.

resources/google/sample_gcp_service_account.json

@@ -0,0 +1,12 @@
+{
+ "type": "service_account",
+ "project_id": "",
+ "private_key_id": "",
+ "private_key": "",
+ "client_email": "",
+ "client_id": "",
+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+ "token_uri": "https://oauth2.googleapis.com/token",
+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+ "client_x509_cert_url": ""
+}

run.sh

@@ -22,6 +22,10 @@ export API_ENDPOINT=$API_ENDPOINT
 export KAFKA_TYPE=$KAFKA_TYPE
 export CONFLUENT_KEY=$CONFLUENT_KEY
 export CONFLUENT_SECRET=$CONFLUENT_SECRET
+export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS
+export KAFKA_ENABLE_AUTO_COMMIT=$KAFKA_ENABLE_AUTO_COMMIT
+export KAFKA_AUTO_COMMIT_INTERVAL_MS=$KAFKA_AUTO_COMMIT_INTERVAL_MS
+export KAFKA_SESSION_TIMEOUT_MS=$KAFKA_SESSION_TIMEOUT_MS
 
 # build
 echo "Compiling."

sample.env

@@ -3,6 +3,7 @@ BOOTSTRAP_SERVER=${DOCKER_HOST_IP}:9092
 API_ENDPOINT=
 DEPOSIT_TOPIC=
 DEPOSIT_GROUP=
+# Bucket name to deposit to (required for all destination types)
 DEPOSIT_BUCKET_NAME=
 DEPOSIT_KEY_NAME=
 AWS_SESSION_TOKEN=
@@ -14,6 +15,12 @@ HEADER_X_API_KEY=
 KAFKA_TYPE=
 CONFLUENT_KEY=
 CONFLUENT_SECRET=
+# Defaults to false
+KAFKA_ENABLE_AUTO_COMMIT= 
+# Defaults to 1000
+KAFKA_AUTO_COMMIT_INTERVAL_MS= 
+# Defaults to 30000
+KAFKA_SESSION_TIMEOUT_MS= 
 
 # MONGODB Variables
 MONGO_IP=${DOCKER_HOST_IP}
@@ -23,3 +30,12 @@ MONGO_DB_PASS=
 MONGO_PORT=27017
 MONGO_URI=mongodb://${MONGO_DB_USER}:${MONGO_DB_PASS}@${MONGO_IP}:${MONGO_PORT}/
 MONGO_COLLECTION_TTL=7 # days
+
+# Destination to deposit messages to
+## Currently supported types are "s3" (Amazon S3 Bucket), "firehose" (Amazon Firehose), or "gcs" (Google Cloud Storage)
+DESTINATION=
+
+# Google Cloud Storage Variables
+## path to service account key file (json format)
+### If GOOGLE_APPLICATION_CREDENTIALS is blank, regardless of destination, it may cause issues when running the S3 depositor with Docker.
+GOOGLE_APPLICATION_CREDENTIALS='./resources/google/sample_gcp_service_account.json'