refactor source harness and GitHub action to add source-mysql

.github/workflows/connector-performance-command.yml

@@ -81,8 +81,9 @@ jobs:
         if: steps.regex.outputs.first_match != github.event.inputs.connector
         run: echo "The connector provided has an invalid format!" && exit 1
       - name: Filter supported connectors
-        if: "${{ github.event.inputs.connector != 'connectors/source-postgres' }}"
-        run: echo "Only connectors/source-postgres currently supported by harness" && exit 1
+        if: "${{ github.event.inputs.connector != 'connectors/source-postgres' && 
+              github.event.inputs.connector != 'connectors/source-mysql' }}"
+        run: echo "Only connectors/source-postgres and source-mysql currently supported by harness" && exit 1
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
@@ -117,8 +118,9 @@ jobs:
         run: |
           echo "Building... ${{github.event.inputs.connector}}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line
-          echo "Running ./gradlew :airbyte-integrations:connectors:source-postgres:build -x check" 
-          ./gradlew :airbyte-integrations:connectors:source-postgres:build -x check
+          connector_name=$(echo ${{ github.event.inputs.connector }} | cut -d / -f 2)
+          echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check" 
+          ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check
       - name: KIND Kubernetes Cluster Setup
         uses: helm/kind-action@v1.4.0
         with:
@@ -133,7 +135,8 @@ jobs:
           suffix: '"}}'
         run: |
           kubectl apply -f ./tools/bin/admin-service-account.yaml
-          kind load docker-image airbyte/source-postgres:dev --name chart-testing
+          connector_name=$(echo $CONN | cut -d / -f 2)
+          kind load docker-image airbyte/$connector_name:dev --name chart-testing
           kind load docker-image airbyte/source-harness:dev --name chart-testing
           export CONNECTOR_IMAGE_NAME=${CONN/connectors/airbyte}:dev
           export DATASET=$DS

airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/performance/Main.java

@@ -19,7 +19,7 @@
 @Slf4j
 public class Main {
 
-  private static final String CREDENTIALS_PATH = "secrets/%s_credentials.json";
+  private static final String CREDENTIALS_PATH = "secrets/%s_%s_credentials.json";
 
   public static void main(final String[] args) {
     log.info("args: {}", Arrays.toString(args));
@@ -32,9 +32,15 @@ public static void main(final String[] args) {
         image = args[0];
         dataset = args[1];
       }
+      default -> {
+        log.info("unexpected arguments");
+        System.exit(1);
+      }
     }
 
-    final Path credsPath = Path.of(CREDENTIALS_PATH.formatted(dataset));
+    final String connector = image.substring(image.indexOf("/") + 1, image.indexOf(":"));
+    log.info("Connector name: {}", connector);
+    final Path credsPath = Path.of(CREDENTIALS_PATH.formatted(connector, dataset));
 
     if (!Files.exists(credsPath)) {
       throw new IllegalStateException("{module-root}/" + credsPath + " not found. Must provide path to a source-harness credentials file.");
@@ -44,7 +50,7 @@ public static void main(final String[] args) {
 
     final JsonNode catalog;
     try {
-      catalog = getCatalog(dataset);
+      catalog = getCatalog(dataset, connector);
     } catch (final IOException ex) {
       throw new IllegalStateException("Failed to read catalog", ex);
     }
@@ -86,9 +92,9 @@ public static void main(final String[] args) {
     System.exit(0);
   }
 
-  static JsonNode getCatalog(final String dataset) throws IOException {
+  static JsonNode getCatalog(final String dataset, final String connector) throws IOException {
     final ObjectMapper objectMapper = new ObjectMapper();
-    final String catalogFilename = "catalogs/%s_catalog.json".formatted(dataset);
+    final String catalogFilename = "catalogs/%s/%s_catalog.json".formatted(connector, dataset);
     final InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(catalogFilename);
     return objectMapper.readTree(is);
   }

airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json

@@ -0,0 +1,161 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "purchases",
+        "namespace": "performance_10m",
+        "json_schema": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "number",
+              "airbyte_type": "integer"
+            },
+            "user_id": {
+              "type": "number",
+              "airbyte_type": "integer"
+            },
+            "product_id": {
+              "type": "number",
+              "airbyte_type": "integer"
+            },
+            "returned_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
+            "purchased_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
+            "added_to_cart_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            }
+          }
+        },
+        "default_cursor_field": [],
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "primary_key": [["id"]],
+      "cursor_field": ["id"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "users",
+        "namespace": "performance_10m",
+        "json_schema": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "number",
+              "airbyte_type": "integer"
+            },
+            "age": {
+              "type": "number",
+              "airbyte_type": "integer"
+            },
+            "name": {
+              "type": "string"
+            },
+            "email": {
+              "type": "string"
+            },
+            "title": {
+              "type": "string"
+            },
+            "gender": {
+              "type": "string"
+            },
+            "height": {
+              "type": "number"
+            },
+            "weight": {
+              "type": "number",
+              "airbyte_type": "integer"
+            },
+            "language": {
+              "type": "string"
+            },
+            "telephone": {
+              "type": "string"
+            },
+            "blood_type": {
+              "type": "string"
+            },
+            "created_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
+            "occupation": {
+              "type": "string"
+            },
+            "updated_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
+            "nationality": {
+              "type": "string"
+            },
+            "academic_degree": {
+              "type": "string"
+            }
+          }
+        },
+        "default_cursor_field": [],
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "primary_key": [["id"]],
+      "cursor_field": ["updated_at"],
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "products",
+        "namespace": "performance_10m",
+        "json_schema": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "number",
+              "airbyte_type": "integer"
+            },
+            "make": {
+              "type": "string"
+            },
+            "year": {
+              "type": "string"
+            },
+            "model": {
+              "type": "string"
+            },
+            "price": {
+              "type": "number"
+            },
+            "created_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            }
+          }
+        },
+        "default_cursor_field": [],
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "primary_key": [["id"]],
+      "cursor_field": ["created_at"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}