Merge branch 'main' into gavin/phoenix-connector

.github/workflows/build-connectors-action.yaml

@@ -0,0 +1,99 @@
+name: Build and Release Database Connector
+
+on:
+  push:
+    branches:
+      - "snowflake/*"
+      - "mysql/*"
+      - "oracle/*"
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        database: [snowflake, mysql, oracle]
+
+    outputs:
+      release_tag: ${{ steps.extract_tag.outputs.tag }}
+
+    permissions:
+      contents: read
+      packages: write # Allows pushing to GHCR
+      id-token: write # Required for authenticating with GHCR
+
+    steps:
+      - name: Checkout repository
+        if: contains(github.ref, matrix.database)
+        uses: actions/checkout@v3
+
+      - name: Extract version from branch name
+        if: contains(github.ref, matrix.database)
+        id: extract_tag
+        run: |
+          # Get the database type from the matrix
+          DB_TYPE=${{ matrix.database }}
+          # Use sed to remove the database prefix and get only the version part
+          VERSION=$(echo "${GITHUB_REF#refs/heads/}" | sed "s|^${DB_TYPE}/||")
+          # Construct the Docker tag
+          echo "docker_tag=ghcr.io/hasura/ndc-jvm-${DB_TYPE}:${VERSION}" >> $GITHUB_OUTPUT
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        if: contains(github.ref, matrix.database)
+        uses: docker/setup-buildx-action@v2
+
+      - name: Cross-platform Docker build and push
+        if: contains(github.ref, matrix.database)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ndc-connector-${{ matrix.database }}.dockerfile
+          push: true
+          tags: ${{ steps.extract_tag.outputs.docker_tag }}
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            JOOQ_PRO_EMAIL=${{ secrets.JOOQ_PRO_EMAIL }}
+            JOOQ_PRO_LICENSE=${{ secrets.JOOQ_PRO_LICENSE }}
+
+  create-release:
+    needs: docker-build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        database: [snowflake, mysql, oracle]
+
+    steps:
+      - name: Checkout repository
+        if: contains(github.ref, matrix.database)
+        uses: actions/checkout@v3
+
+      - name: Update dockerImage in connector-metadata.yaml
+        if: contains(github.ref, matrix.database)
+        run: |
+          # Use the full Docker tag from the docker-build job output
+          sed -i "s|^  dockerImage:.*|  dockerImage: \"${{ needs.docker-build.outputs.docker_tag }}\"|" ndc-connector-${{ matrix.database }}/.hasura-connector/connector-metadata.yaml
+
+      - name: Compress Hasura Connector Metadata
+        if: contains(github.ref, matrix.database)
+        run: |
+          cd ndc-connector-${{ matrix.database }}
+          tar -czf package.tar.gz ./.hasura-connector
+
+      - name: Upload package.tar.gz to GitHub Release
+        if: contains(github.ref, matrix.database)
+        uses: actions/upload-release-asset@v1
+        with:
+          tag_name: ${{ needs.docker-build.outputs.tag }} # Use the correct tag output
+          upload_url: ${{ steps.release.outputs.upload_url }}
+          asset_path: ndc-connector-${{ matrix.database }}/package.tar.gz
+          asset_name: package-${{ matrix.database }}.tar.gz
+          asset_content_type: application/gzip
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

create-github-release.sh

@@ -2,15 +2,15 @@
 
 # Check if the correct number of arguments is provided
 if [ "$#" -ne 2 ]; then
-  echo "Usage: $0 <version> <subdir>"
+  echo "Usage: $0 <subdir> <version>"
   exit 1
 fi
 
 # Variables
 OWNER="hasura"
 REPO="ndc-jvm-mono"
-VERSION="$1"  # Version passed as the first argument
-SUBDIR="$2"   # Subdirectory passed as the second argument
+SUBDIR="$1"   # Subdirectory passed as the second argument
+VERSION="$2"  # Version passed as the first argument
 
 # Create tag, release name, and description
 TAG="${SUBDIR#ndc-connector-}/${VERSION}"  # Create tag like oracle/v1.0.0
@@ -36,4 +36,4 @@ if [ $? -eq 0 ]; then
   echo "Release ${RELEASE_NAME} created and file uploaded successfully for ${SUBDIR}."
 else
   echo "Failed to create release ${RELEASE_NAME} or upload file for ${SUBDIR}."
-fi
+fi

gradle.properties

@@ -1,4 +1,4 @@
-version=0.1.0
+version=1.0.1
 
 #Gradle properties
 quarkusPluginId=io.quarkus

ndc-app/src/main/kotlin/io/hasura/ndc/app/application/Filters.kt

@@ -15,7 +15,8 @@ class Filters {
     @ServerRequestFilter(priority = 0)
     fun logBodyFilter(info: UriInfo, request: HttpServerRequest, ctx: ContainerRequestContext) {
         request.body { b ->
-            logger.debug("INCOMING IR: ${b.result()}")
+            logger.debug("INCOMING IR:")
+            logger.debug(b.result())
         }
     }
 }

ndc-app/src/main/kotlin/io/hasura/ndc/app/controllers/DataConnectorResource.kt

@@ -22,7 +22,7 @@ class DataConnectorResource @Inject constructor(
         val canConnectToDB = dataConnectorService.runHealthCheckQuery()
         if (canConnectToDB) {
             return Response
-                .status(Response.Status.NO_CONTENT)
+                .status(Response.Status.OK)
                 .build()
         } else {
             throw RuntimeException("Unable to connect to DB")

ndc-app/src/main/kotlin/io/hasura/ndc/app/services/AgroalDataSourceService.kt

@@ -9,6 +9,7 @@ import io.agroal.api.security.SimplePassword
 import io.hasura.ndc.common.ConnectorConfiguration
 import io.opentelemetry.instrumentation.annotations.WithSpan
 import io.opentelemetry.instrumentation.jdbc.datasource.OpenTelemetryDataSource
+import io.quarkus.agroal.runtime.AgroalOpenTelemetryWrapper
 import io.quarkus.agroal.runtime.OpenTelemetryAgroalDataSource
 import io.smallrye.config.ConfigMapping
 import io.smallrye.config.WithDefault

ndc-app/src/main/kotlin/io/hasura/ndc/app/services/dataConnectors/BaseDataConnectorService.kt

@@ -9,7 +9,6 @@ import io.hasura.ndc.app.interfaces.ISchemaGenerator
 import io.hasura.ndc.common.ConnectorConfiguration
 import io.hasura.ndc.app.models.ExplainResponse
 import io.hasura.ndc.app.services.AgroalDataSourceService
-import io.hasura.ndc.app.services.ConnectorConfigurationLoader
 import io.opentelemetry.api.trace.Tracer
 import io.opentelemetry.instrumentation.annotations.WithSpan
 import jakarta.enterprise.inject.Produces
@@ -106,14 +105,14 @@ abstract class BaseDataConnectorService(
 
     @WithSpan
     open fun mkDSLCtx(): DSLContext {
-        val config = ConnectorConfigurationLoader.config
+        val config = ConnectorConfiguration.Loader.config
         val ds = dataSourceProvider.getDataSource(config)
         return DSL.using(ds, jooqDialect, jooqSettings)
     }
 
     @WithSpan
     override fun getSchema(): SchemaResponse {
-        return schemaGenerator.getSchema(ConnectorConfigurationLoader.config)
+        return schemaGenerator.getSchema(ConnectorConfiguration.Loader.config)
     }
 
 

ndc-cli/src/main/kotlin/io/hasura/cli/main.kt

@@ -1,6 +1,7 @@
 package io.hasura.cli
 
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import io.hasura.ndc.common.ConnectorConfiguration
 import picocli.CommandLine
 import picocli.CommandLine.*
 import java.io.File
@@ -54,11 +55,22 @@ class CLI {
             names = ["-s", "--schemas"],
             arity = "0..*",
             split = ",",
-            defaultValue = "",
             description = ["Comma-separated list of schemas to introspect"]
         )
-        schemas: List<String> = emptyList()
+        schemas: List<String>?
     ) {
+        val file = File(outfile)
+
+        println("Checking for configuration file at ${file.absolutePath}")
+        val existingConfig = file.let {
+            if (it.exists()) {
+                println("Existing configuration file detected")
+                mapper.readValue(it, ConnectorConfiguration::class.java)
+            } else {
+                println("Non-existent or empty configuration file detected")
+                ConnectorConfiguration()
+            }
+        }
 
         val configGenerator = when (database) {
             DatabaseType.ORACLE -> OracleConfigGenerator
@@ -67,20 +79,23 @@ class CLI {
             DatabaseType.PHOENIX -> PhoenixConfigGenerator
         }
 
-        val config = configGenerator.getConfig(
+        println("Generating configuration for $database database...")
+        val introspectedConfig = configGenerator.getConfig(
             jdbcUrl = jdbcUrl,
-            schemas = schemas
+            schemas = schemas ?: emptyList()
+        )
+        val mergedConfigWithNativeQueries = introspectedConfig.copy(
+            nativeQueries = existingConfig.nativeQueries
         )
 
-        val file = File(outfile)
         try {
-            file.createNewFile()
-            mapper.writerWithDefaultPrettyPrinter().writeValue(file, config)
+            println("Writing configuration to ${file.absolutePath}")
+            mapper.writerWithDefaultPrettyPrinter().writeValue(file, mergedConfigWithNativeQueries)
         } catch (e: Exception) {
             println("Error writing configuration to file: ${e.message}")
 
             val parentDir = file.parentFile
-            val permissions =  Files.getPosixFilePermissions(parentDir.toPath())
+            val permissions = Files.getPosixFilePermissions(parentDir.toPath())
             val posixPermissions = PosixFilePermissions.toString(permissions)
 
             println("Current user: ${System.getProperty("user.name")}")
@@ -104,4 +119,3 @@ class CLI {
         }
     }
 }
-

ndc-connector-mysql.dockerfile

@@ -1,7 +1,13 @@
 # Build stage
 FROM registry.access.redhat.com/ubi9/openjdk-21:1.20-2 AS build
 
+ARG JOOQ_PRO_EMAIL
+ARG JOOQ_PRO_LICENSE
+
 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en'
+ENV JOOQ_PRO_EMAIL=${JOOQ_PRO_EMAIL}
+ENV JOOQ_PRO_LICENSE=${JOOQ_PRO_LICENSE}
+
 WORKDIR /build
 COPY . /build
 

ndc-connector-mysql/.hasura-connector/connector-metadata.yaml

@@ -1,6 +1,6 @@
 packagingDefinition:
   type: PrebuiltDockerImage
-  dockerImage: "ghcr.io/hasura/ndc-jvm-mysql:v0.1.0"
+  dockerImage: "ghcr.io/hasura/ndc-jvm-mysql:v1.0.3"
 supportedEnvironmentVariables:
   - name: JDBC_URL
     description: "The JDBC URL to connect to the database"
@@ -9,7 +9,7 @@ commands:
     docker run \
       -e HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH \
       -v ${HASURA_PLUGIN_CONNECTOR_CONTEXT_PATH}:/app/output \
-      ghcr.io/hasura/ndc-jvm-cli:v0.1.0 update $JDBC_URL \
+      ghcr.io/hasura/ndc-jvm-cli:v0.1.3 update $JDBC_URL \
         --database MYSQL \
         --schemas $JDBC_SCHEMAS \
         --outfile /app/output/configuration.json