Faker V0.3 - CDK, consistent naming, determinism, display name, and S…

…AT (#19490) * better comment to show sync types * WIP - basics in place * test cleanup * emit `AirbyteEstimateTraceMessage` * assert estimates emitted * unit tests passing * format * flake * check tests * add newline * lint * check in with new file name * bump acceptance test yml * acceptance test progress * deterministic purchases * SAT making progress * better disable * adjust yaml * change abnormal_state.json format * per-stream state * products for abnormal state * products have state + incremental * purchases state = total_user_records * update docs and name * disable proper version * typo * try disable 0.2.1 * kwargs * v1.0.0 * lint * backward_compatibility_tests_config * auto-bump connector version Co-authored-by: alafanechere <augustin.lafanechere@gmail.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Dec 1, 2022 · 1e160b2 · 1e160b2
1 parent b4a334e
commit 1e160b2
Show file tree

Hide file tree

Showing 23 changed files with 739 additions and 427 deletions.
diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py
@@ -226,7 +226,7 @@ def _read_incremental(
             sync_mode=SyncMode.incremental,
             stream_state=stream_state,
         )
-        logger.debug(f"Processing stream slices for {stream_name}", extra={"stream_slices": slices})
+        logger.debug(f"Processing stream slices for {stream_name} (sync_mode: incremental)", extra={"stream_slices": slices})
 
         total_records_counter = 0
         has_slices = False
@@ -276,7 +276,9 @@ def _read_full_refresh(
         internal_config: InternalConfig,
     ) -> Iterator[AirbyteMessage]:
         slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=configured_stream.cursor_field)
-        logger.debug(f"Processing stream slices for {configured_stream.stream.name}", extra={"stream_slices": slices})
+        logger.debug(
+            f"Processing stream slices for {configured_stream.stream.name} (sync_mode: full_refresh)", extra={"stream_slices": slices}
+        )
         total_records_counter = 0
         for _slice in slices:
             logger.debug("Processing stream slice", extra={"slice": _slice})

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -457,10 +457,10 @@
   icon: facebook.svg
   sourceType: api
   releaseStage: alpha
-- name: "Sample Data (faker)"
+- name: Sample Data (Faker)
   sourceDefinitionId: dfd88b22-b603-4c3d-aad7-3701784586b1
   dockerRepository: airbyte/source-faker
-  dockerImageTag: 0.2.1
+  dockerImageTag: 1.0.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/faker
   sourceType: api
   releaseStage: alpha

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3832,7 +3832,7 @@
         oauthFlowInitParameters: []
         oauthFlowOutputParameters:
         - - "access_token"
-- dockerImage: "airbyte/source-faker:0.2.1"
+- dockerImage: "airbyte/source-faker:1.0.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/faker"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile
@@ -34,5 +34,5 @@ COPY source_faker ./source_faker
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.1
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-faker
diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml
@@ -1,29 +1,40 @@
-# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
-# for more information about how to configure these tests
 connector_image: airbyte/source-faker:dev
-tests:
-  spec:
-    - spec_path: "source_faker/spec.json"
+test_strictness_level: high
+acceptance_tests:
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: secrets/config.json
+        status: succeed
+      - config_path: integration_tests/invalid_config.json
+        status: failed
+  spec:
+    tests:
+      - spec_path: source_faker/spec.json
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: secrets/config.json
+        backward_compatibility_tests_config:
+          disable_for_version: "0.2.1"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: []
-      expect_records:
-        path: "integration_tests/expected_records.txt"
-        extra_fields: no
-        exact_order: yes
-        extra_records: no
+    tests:
+      - config_path: secrets/config.json
+        empty_streams: []
+        expect_records:
+          path: integration_tests/expected_records.jsonl
+          exact_order: true
+          extra_fields: false
+          extra_records: false
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      ignored_fields:
-        "Users":
-          - created_at
-          - updated_at
+    tests:
+      - config_path: secrets/config.json
+        configured_catalog_path: integration_tests/configured_catalog.json
+        ignored_fields:
+          users:
+            - created_at
+            - updated_at
+  incremental:
+    tests:
+      - config_path: secrets/config.json
+        configured_catalog_path: integration_tests/configured_catalog.json
+        future_state:
+          future_state_path: integration_tests/abnormal_state.json
diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json
@@ -1,5 +1,35 @@
-{
-  "Users": {
-    "cursor": -1
+[
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "id": 11
+      },
+      "stream_descriptor": {
+        "name": "users"
+      }
+    }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "user_id": 11
+      },
+      "stream_descriptor": {
+        "name": "purchases"
+      }
+    }
+  },
+  {
+    "type": "STREAM",
+    "stream": {
+      "stream_state": {
+        "id": 101
+      },
+      "stream_descriptor": {
+        "name": "products"
+      }
+    }
   }
-}
+]
diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json b/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json
@@ -1,7 +1,7 @@
 {
   "streams": [
     {
-      "name": "Users",
+      "name": "users",
       "json_schema": {
         "$schema": "http://json-schema.org/draft-07/schema#",
         "type": "object",

diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json
@@ -2,7 +2,7 @@
   "streams": [
     {
       "stream": {
-        "name": "Users",
+        "name": "users",
         "json_schema": {
           "$schema": "http://json-schema.org/draft-07/schema#",
           "type": "object",
@@ -37,7 +37,63 @@
         "source_defined_cursor": true,
         "default_cursor_field": ["created_at"]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "purchases",
+        "json_schema": {
+          "properties": {
+            "id": { "type": "number" },
+            "user_id": { "type": "number" },
+            "product_id": { "type": "number" },
+            "added_to_cart_at": {
+              "type": ["null", "string"],
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
+            "purchased_at": {
+              "type": ["null", "string"],
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            },
+            "returned_at": {
+              "type": ["null", "string"],
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            }
+          }
+        },
+        "supported_sync_modes": ["incremental", "full_refresh"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created_at"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "products",
+        "json_schema": {
+          "properties": {
+            "id": { "type": "number" },
+            "make": { "type": "string" },
+            "model": { "type": "string" },
+            "year": { "type": "number" },
+            "price": { "type": "number" },
+            "created_at": {
+              "type": "string",
+              "format": "date-time",
+              "airbyte_type": "timestamp_with_timezone"
+            }
+          }
+        },
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created_at"]
+      },
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     }
   ]