From 544eedebef9f09cd43175565270343c0f9d7012f Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Mon, 10 Apr 2023 21:35:39 -0700 Subject: [PATCH 1/9] refactor to add source-mysql --- .../connector-performance-command.yml | 14 +- .../integrations/performance/Main.java | 16 +- .../main/resources/catalogs/10m_catalog.json | 161 ------------------ .../main/resources/catalogs/20m_catalog.json | 161 ------------------ .../catalogs/bottleneck_stream1_catalog.json | 37 ---- .../{ => source-mysql}/1m_catalog.json | 6 +- 6 files changed, 23 insertions(+), 372 deletions(-) delete mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/10m_catalog.json delete mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/20m_catalog.json delete mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/bottleneck_stream1_catalog.json rename airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/{ => source-mysql}/1m_catalog.json (97%) diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index c458e0895758..bfcadc8be370 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -81,8 +81,10 @@ jobs: if: steps.regex.outputs.first_match != github.event.inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors - if: "${{ github.event.inputs.connector != 'connectors/source-postgres' }}" - run: echo "Only connectors/source-postgres currently supported by harness" && exit 1 + if: | + "${{ github.event.inputs.connector != 'connectors/source-postgres' + && github.event.inputs.connector != 'connectors/source-mysql' }}" + run: echo "Only connectors/source-postgres and source-mysql currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 with: @@ -117,8 +119,9 @@ jobs: run: | echo "Building... ${{github.event.inputs.connector}}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line - echo "Running ./gradlew :airbyte-integrations:connectors:source-postgres:build -x check" - ./gradlew :airbyte-integrations:connectors:source-postgres:build -x check + connector_name=$(echo ${{ github.event.inputs.connector }} | cut -d / -f 2) + echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check" + ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check - name: KIND Kubernetes Cluster Setup uses: helm/kind-action@v1.4.0 with: @@ -133,7 +136,8 @@ jobs: suffix: '"}}' run: | kubectl apply -f ./tools/bin/admin-service-account.yaml - kind load docker-image airbyte/source-postgres:dev --name chart-testing + connector_name=$(echo $CONN | cut -d / -f 2) + kind load docker-image airbyte/$connector_name:dev --name chart-testing kind load docker-image airbyte/source-harness:dev --name chart-testing export CONNECTOR_IMAGE_NAME=${CONN/connectors/airbyte}:dev export DATASET=$DS diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/performance/Main.java b/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/performance/Main.java index 7ceb761128a7..155cd31a79fe 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/performance/Main.java +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/performance/Main.java @@ -19,7 +19,7 @@ @Slf4j public class Main { - private static final String CREDENTIALS_PATH = "secrets/%s_credentials.json"; + private static final String CREDENTIALS_PATH = "secrets/%s_%s_credentials.json"; public static void main(final String[] args) { log.info("args: {}", Arrays.toString(args)); @@ -32,9 +32,15 @@ public static void main(final String[] args) { image = args[0]; dataset = args[1]; } + default -> { + log.info("unexpected arguments"); + System.exit(1); + } } - final Path credsPath = Path.of(CREDENTIALS_PATH.formatted(dataset)); + final String connector = image.substring(image.indexOf("/") + 1, image.indexOf(":")); + log.info("Connector name: {}", connector); + final Path credsPath = Path.of(CREDENTIALS_PATH.formatted(connector, dataset)); if (!Files.exists(credsPath)) { throw new IllegalStateException("{module-root}/" + credsPath + " not found. Must provide path to a source-harness credentials file."); @@ -44,7 +50,7 @@ public static void main(final String[] args) { final JsonNode catalog; try { - catalog = getCatalog(dataset); + catalog = getCatalog(dataset, connector); } catch (final IOException ex) { throw new IllegalStateException("Failed to read catalog", ex); } @@ -86,9 +92,9 @@ public static void main(final String[] args) { System.exit(0); } - static JsonNode getCatalog(final String dataset) throws IOException { + static JsonNode getCatalog(final String dataset, final String connector) throws IOException { final ObjectMapper objectMapper = new ObjectMapper(); - final String catalogFilename = "catalogs/%s_catalog.json".formatted(dataset); + final String catalogFilename = "catalogs/%s/%s_catalog.json".formatted(connector, dataset); final InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(catalogFilename); return objectMapper.readTree(is); } diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/10m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/10m_catalog.json deleted file mode 100644 index 07b22645a2a9..000000000000 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/10m_catalog.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "purchases", - "namespace": "10m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "user_id": { - "type": "number", - "airbyte_type": "integer" - }, - "product_id": { - "type": "number", - "airbyte_type": "integer" - }, - "returned_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "purchased_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "added_to_cart_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["id"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "users", - "namespace": "10m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "age": { - "type": "number", - "airbyte_type": "integer" - }, - "name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "title": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "height": { - "type": "number" - }, - "weight": { - "type": "number", - "airbyte_type": "integer" - }, - "language": { - "type": "string" - }, - "telephone": { - "type": "string" - }, - "blood_type": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "occupation": { - "type": "string" - }, - "updated_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "nationality": { - "type": "string" - }, - "academic_degree": { - "type": "string" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["updated_at"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "products", - "namespace": "10m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "make": { - "type": "string" - }, - "year": { - "type": "string" - }, - "model": { - "type": "string" - }, - "price": { - "type": "number" - }, - "created_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["created_at"], - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/20m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/20m_catalog.json deleted file mode 100644 index 93302af014f6..000000000000 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/20m_catalog.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "purchases", - "namespace": "20m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "user_id": { - "type": "number", - "airbyte_type": "integer" - }, - "product_id": { - "type": "number", - "airbyte_type": "integer" - }, - "returned_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "purchased_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "added_to_cart_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["id"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "users", - "namespace": "20m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "age": { - "type": "number", - "airbyte_type": "integer" - }, - "name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "title": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "height": { - "type": "number" - }, - "weight": { - "type": "number", - "airbyte_type": "integer" - }, - "language": { - "type": "string" - }, - "telephone": { - "type": "string" - }, - "blood_type": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "occupation": { - "type": "string" - }, - "updated_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "nationality": { - "type": "string" - }, - "academic_degree": { - "type": "string" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["updated_at"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "products", - "namespace": "20m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "make": { - "type": "string" - }, - "year": { - "type": "string" - }, - "model": { - "type": "string" - }, - "price": { - "type": "number" - }, - "created_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["created_at"], - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/bottleneck_stream1_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/bottleneck_stream1_catalog.json deleted file mode 100644 index 7c2a48f1f4b7..000000000000 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/bottleneck_stream1_catalog.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "stream1", - "namespace": "postgres", - "json_schema": { - "type": "object", - "properties": { - "field1": { - "type": "string" - }, - "field2": { - "type": "string" - }, - "field3": { - "type": "string" - }, - "field4": { - "type": "string" - }, - "field5": { - "type": "string" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["field1"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["field1"]], - "cursor_field": ["field1"], - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/1m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json similarity index 97% rename from airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/1m_catalog.json rename to airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json index 5e46b7b5d5cb..d658a8319c59 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/1m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json @@ -3,7 +3,7 @@ { "stream": { "name": "purchases", - "namespace": "1m_users", + "namespace": "benchmark", "json_schema": { "type": "object", "properties": { @@ -48,7 +48,7 @@ { "stream": { "name": "users", - "namespace": "1m_users", + "namespace": "benchmark", "json_schema": { "type": "object", "properties": { @@ -121,7 +121,7 @@ { "stream": { "name": "products", - "namespace": "1m_users", + "namespace": "benchmark", "json_schema": { "type": "object", "properties": { From d69666614b68090112c5f3d01037f497fa793d82 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Mon, 10 Apr 2023 21:47:23 -0700 Subject: [PATCH 2/9] refactor to add source-mysql --- .../connector-performance-command.yml | 4 +- .../catalogs/source-postgres/10m_catalog.json | 161 ++++++++++++++++++ .../catalogs/source-postgres/1m_catalog.json | 161 ++++++++++++++++++ .../catalogs/source-postgres/20m_catalog.json | 161 ++++++++++++++++++ .../bottleneck_stream1_catalog.json | 37 ++++ 5 files changed, 522 insertions(+), 2 deletions(-) create mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json create mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json create mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json create mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/bottleneck_stream1_catalog.json diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index bfcadc8be370..a0882d8589b9 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -82,8 +82,8 @@ jobs: run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors if: | - "${{ github.event.inputs.connector != 'connectors/source-postgres' - && github.event.inputs.connector != 'connectors/source-mysql' }}" + "${{ (github.event.inputs.connector != 'connectors/source-postgres') + && (github.event.inputs.connector != 'connectors/source-mysql') }}" run: echo "Only connectors/source-postgres and source-mysql currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json new file mode 100644 index 000000000000..07b22645a2a9 --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json @@ -0,0 +1,161 @@ +{ + "streams": [ + { + "stream": { + "name": "purchases", + "namespace": "10m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "user_id": { + "type": "number", + "airbyte_type": "integer" + }, + "product_id": { + "type": "number", + "airbyte_type": "integer" + }, + "returned_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "purchased_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_to_cart_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "namespace": "10m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "age": { + "type": "number", + "airbyte_type": "integer" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "title": { + "type": "string" + }, + "gender": { + "type": "string" + }, + "height": { + "type": "number" + }, + "weight": { + "type": "number", + "airbyte_type": "integer" + }, + "language": { + "type": "string" + }, + "telephone": { + "type": "string" + }, + "blood_type": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "occupation": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "nationality": { + "type": "string" + }, + "academic_degree": { + "type": "string" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "namespace": "10m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "make": { + "type": "string" + }, + "year": { + "type": "string" + }, + "model": { + "type": "string" + }, + "price": { + "type": "number" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["created_at"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json new file mode 100644 index 000000000000..5e46b7b5d5cb --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json @@ -0,0 +1,161 @@ +{ + "streams": [ + { + "stream": { + "name": "purchases", + "namespace": "1m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "user_id": { + "type": "number", + "airbyte_type": "integer" + }, + "product_id": { + "type": "number", + "airbyte_type": "integer" + }, + "returned_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "purchased_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_to_cart_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "namespace": "1m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "age": { + "type": "number", + "airbyte_type": "integer" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "title": { + "type": "string" + }, + "gender": { + "type": "string" + }, + "height": { + "type": "number" + }, + "weight": { + "type": "number", + "airbyte_type": "integer" + }, + "language": { + "type": "string" + }, + "telephone": { + "type": "string" + }, + "blood_type": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "occupation": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "nationality": { + "type": "string" + }, + "academic_degree": { + "type": "string" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "namespace": "1m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "make": { + "type": "string" + }, + "year": { + "type": "string" + }, + "model": { + "type": "string" + }, + "price": { + "type": "number" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["created_at"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json new file mode 100644 index 000000000000..93302af014f6 --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json @@ -0,0 +1,161 @@ +{ + "streams": [ + { + "stream": { + "name": "purchases", + "namespace": "20m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "user_id": { + "type": "number", + "airbyte_type": "integer" + }, + "product_id": { + "type": "number", + "airbyte_type": "integer" + }, + "returned_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "purchased_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_to_cart_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "namespace": "20m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "age": { + "type": "number", + "airbyte_type": "integer" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "title": { + "type": "string" + }, + "gender": { + "type": "string" + }, + "height": { + "type": "number" + }, + "weight": { + "type": "number", + "airbyte_type": "integer" + }, + "language": { + "type": "string" + }, + "telephone": { + "type": "string" + }, + "blood_type": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "occupation": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "nationality": { + "type": "string" + }, + "academic_degree": { + "type": "string" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "namespace": "20m_users", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "make": { + "type": "string" + }, + "year": { + "type": "string" + }, + "model": { + "type": "string" + }, + "price": { + "type": "number" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["created_at"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/bottleneck_stream1_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/bottleneck_stream1_catalog.json new file mode 100644 index 000000000000..7c2a48f1f4b7 --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/bottleneck_stream1_catalog.json @@ -0,0 +1,37 @@ +{ + "streams": [ + { + "stream": { + "name": "stream1", + "namespace": "postgres", + "json_schema": { + "type": "object", + "properties": { + "field1": { + "type": "string" + }, + "field2": { + "type": "string" + }, + "field3": { + "type": "string" + }, + "field4": { + "type": "string" + }, + "field5": { + "type": "string" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["field1"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["field1"]], + "cursor_field": ["field1"], + "destination_sync_mode": "append" + } + ] +} From cc21a46d6a91edd9c330c526e787efbf3a46ee3c Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Mon, 10 Apr 2023 21:52:28 -0700 Subject: [PATCH 3/9] refactor to add source-mysql --- .github/workflows/connector-performance-command.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index a0882d8589b9..56ac335e8acb 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -82,8 +82,8 @@ jobs: run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors if: | - "${{ (github.event.inputs.connector != 'connectors/source-postgres') - && (github.event.inputs.connector != 'connectors/source-mysql') }}" + ${{ (github.event.inputs.connector != 'connectors/source-postgres') + && (github.event.inputs.connector != 'connectors/source-mysql') }} run: echo "Only connectors/source-postgres and source-mysql currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 From 21bb3354aba0d3363b161fa8ab85d4b2871e6bca Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Mon, 10 Apr 2023 22:00:31 -0700 Subject: [PATCH 4/9] test --- .github/workflows/connector-performance-command.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index 56ac335e8acb..de0a3b1b21f7 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -81,9 +81,7 @@ jobs: if: steps.regex.outputs.first_match != github.event.inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors - if: | - ${{ (github.event.inputs.connector != 'connectors/source-postgres') - && (github.event.inputs.connector != 'connectors/source-mysql') }} + if: "${{ github.event.inputs.connector != 'connectors/source-mysql' }}" run: echo "Only connectors/source-postgres and source-mysql currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 From b89a6a32306ab5a6613cdcac9b2af6b6ed5820ba Mon Sep 17 00:00:00 2001 From: rodireich Date: Tue, 11 Apr 2023 05:21:40 +0000 Subject: [PATCH 5/9] Automated Change --- .../init-oss/src/main/resources/seed/oss_catalog.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json index fb127e43455a..0b943e148dd4 100644 --- a/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json +++ b/airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json @@ -12594,11 +12594,7 @@ "type": "string", "title": "Reader Options", "description": "This should be a string in JSON format. It depends on the chosen file format to provide additional options and tune its behavior.", - "examples": [ "{}", "{\"sep\": \" \"}", { - "sep": "\t", - "header": 0, - "names": [ "column1", "column2" ] - } ] + "examples": [ "{}", "{\"sep\": \" \"}", "{\"sep\": \"\t\", \"header\": 0, \"names\": [\"column1\", \"column2\"] }" ] }, "url": { "type": "string", From 58eca84b017eaef5af2bdb8028f3064fcd052fd7 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Tue, 11 Apr 2023 22:28:31 -0700 Subject: [PATCH 6/9] fix if condition --- .github/workflows/connector-performance-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index de0a3b1b21f7..1c05015c5272 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -81,7 +81,7 @@ jobs: if: steps.regex.outputs.first_match != github.event.inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors - if: "${{ github.event.inputs.connector != 'connectors/source-mysql' }}" + if: "${{ github.event.inputs.connector != 'connectors/source-postgres' && github.event.inputs.connector != 'connectors/source-mysql' }}" run: echo "Only connectors/source-postgres and source-mysql currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 From 6330b78fa6d78885a40a10479775685b8f080aca Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Tue, 11 Apr 2023 22:40:21 -0700 Subject: [PATCH 7/9] try multiline --- .github/workflows/connector-performance-command.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index 1c05015c5272..5d43370ff921 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -81,7 +81,8 @@ jobs: if: steps.regex.outputs.first_match != github.event.inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors - if: "${{ github.event.inputs.connector != 'connectors/source-postgres' && github.event.inputs.connector != 'connectors/source-mysql' }}" + if: "${{ github.event.inputs.connector != 'connectors/source-postgres' && + github.event.inputs.connector != 'connectors/source-mysql' }}" run: echo "Only connectors/source-postgres and source-mysql currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 From 4981bcdd3ca92d84b3ceb9a92e93675f325dd0f1 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Wed, 12 Apr 2023 22:44:03 -0700 Subject: [PATCH 8/9] new db --- .../main/resources/catalogs/source-mysql/1m_catalog.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json index d658a8319c59..ad274523846a 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json @@ -3,7 +3,7 @@ { "stream": { "name": "purchases", - "namespace": "benchmark", + "namespace": "performance_1m", "json_schema": { "type": "object", "properties": { @@ -48,7 +48,7 @@ { "stream": { "name": "users", - "namespace": "benchmark", + "namespace": "performance_1m", "json_schema": { "type": "object", "properties": { @@ -121,7 +121,7 @@ { "stream": { "name": "products", - "namespace": "benchmark", + "namespace": "performance_1m", "json_schema": { "type": "object", "properties": { From 88731e5d8fb30a943e8a72e4957fcdc51244af0e Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Wed, 12 Apr 2023 23:07:50 -0700 Subject: [PATCH 9/9] add mysql catalogs --- .../catalogs/source-mysql/10m_catalog.json | 161 ++++++++++++++++++ .../catalogs/source-mysql/20m_catalog.json | 161 ++++++++++++++++++ .../bottleneck_stream1_catalog.json | 37 ++++ 3 files changed, 359 insertions(+) create mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json create mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json create mode 100644 airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/bottleneck_stream1_catalog.json diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json new file mode 100644 index 000000000000..75d4fc01be8f --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json @@ -0,0 +1,161 @@ +{ + "streams": [ + { + "stream": { + "name": "purchases", + "namespace": "performance_10m", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "user_id": { + "type": "number", + "airbyte_type": "integer" + }, + "product_id": { + "type": "number", + "airbyte_type": "integer" + }, + "returned_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "purchased_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_to_cart_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "namespace": "performance_10m", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "age": { + "type": "number", + "airbyte_type": "integer" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "title": { + "type": "string" + }, + "gender": { + "type": "string" + }, + "height": { + "type": "number" + }, + "weight": { + "type": "number", + "airbyte_type": "integer" + }, + "language": { + "type": "string" + }, + "telephone": { + "type": "string" + }, + "blood_type": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "occupation": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "nationality": { + "type": "string" + }, + "academic_degree": { + "type": "string" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "namespace": "performance_10m", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "make": { + "type": "string" + }, + "year": { + "type": "string" + }, + "model": { + "type": "string" + }, + "price": { + "type": "number" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["created_at"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json new file mode 100644 index 000000000000..55c38e69c72f --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json @@ -0,0 +1,161 @@ +{ + "streams": [ + { + "stream": { + "name": "purchases", + "namespace": "performance_20m", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "user_id": { + "type": "number", + "airbyte_type": "integer" + }, + "product_id": { + "type": "number", + "airbyte_type": "integer" + }, + "returned_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "purchased_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "added_to_cart_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "namespace": "performance_20m", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "age": { + "type": "number", + "airbyte_type": "integer" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "title": { + "type": "string" + }, + "gender": { + "type": "string" + }, + "height": { + "type": "number" + }, + "weight": { + "type": "number", + "airbyte_type": "integer" + }, + "language": { + "type": "string" + }, + "telephone": { + "type": "string" + }, + "blood_type": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "occupation": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "nationality": { + "type": "string" + }, + "academic_degree": { + "type": "string" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "namespace": "performance_20m", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": "number", + "airbyte_type": "integer" + }, + "make": { + "type": "string" + }, + "year": { + "type": "string" + }, + "model": { + "type": "string" + }, + "price": { + "type": "number" + }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["created_at"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/bottleneck_stream1_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/bottleneck_stream1_catalog.json new file mode 100644 index 000000000000..6e8979b20a2f --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/bottleneck_stream1_catalog.json @@ -0,0 +1,37 @@ +{ + "streams": [ + { + "stream": { + "name": "stream1", + "namespace": "performance_bottleneck", + "json_schema": { + "type": "object", + "properties": { + "field1": { + "type": "string" + }, + "field2": { + "type": "string" + }, + "field3": { + "type": "string" + }, + "field4": { + "type": "string" + }, + "field5": { + "type": "string" + } + } + }, + "default_cursor_field": [], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["field1"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["field1"]], + "cursor_field": ["field1"], + "destination_sync_mode": "append" + } + ] +}