🎉 Update hardway tutorial & generic source template to use SAT (#11908)

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py

@@ -256,8 +256,8 @@ def _validate_records_structure(records: List[AirbyteRecordMessage], configured_
         just running schema validation is not enough case schema could have
         additionalProperties parameter set to true and no required fields
         therefore any arbitrary object would pass schema validation.
-        This method is here to catch those cases by extracting all the pathes
-        from the object and compare it to pathes expected from jsonschema. If
+        This method is here to catch those cases by extracting all the paths
+        from the object and compare it to paths expected from jsonschema. If
         there no common pathes then raise an alert.
 
         :param records: List of airbyte record messages gathered from connector instances.

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py

@@ -168,7 +168,7 @@ def _traverse_obj_and_get_path(obj, path=""):
 
 def get_expected_schema_structure(schema: dict, annotate_one_of: bool = False) -> List[str]:
     """
-    Travers through json schema and compose list of property keys that object expected to have.
+    Traverse through json schema and compose list of property keys that object expected to have.
     :param annotate_one_of: Generate one_of index in path
     :param schema: jsonschema to get expected paths
     :returns list of object property keys paths

airbyte-integrations/connector-templates/generator/plopfile.js

@@ -175,12 +175,6 @@ module.exports = function (plop) {
           base: genericSourceInputRoot,
           templateFiles: `${genericSourceInputRoot}/**/**`,
         },
-        {
-          type:'add',
-          abortOnFail: true,
-          templateFile: `${genericSourceInputRoot}/.gitignore.hbs`,
-          path: `${genericSourceOutputRoot}/.gitignore`
-        },
         {type: 'emitSuccess', outputPath: genericSourceOutputRoot}
       ]
     });

airbyte-integrations/connector-templates/source-generic/Dockerfile

@@ -1,7 +1,7 @@
 FROM scratch
 
 ## TODO Add your dockerfile instructions here
-## TODO uncomment the below line
+## TODO uncomment the below line. This is required for Kubernetes compatibility.
 # ENV AIRBYTE_ENTRYPOINT="update this with the command you use for an entrypoint"
 
 # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile.

airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml

@@ -0,0 +1,25 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-{{dashCase name}}:dev
+tests:
+  spec:
+    - spec_path: "spec.json"
+      config_path: "secrets/valid_config.json" # TODO add this file
+  connection:
+    - config_path: "secrets/valid_config.json" # TODO add this file
+      status: "succeed"
+    - config_path: "secrets/invalid_config.json" # TODO add this file
+      status: "failed"
+  discovery:
+    - config_path: "secrets/valid_config.json"
+  basic_read:
+    - config_path: "secrets/valid_config.json"
+      configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file
+      empty_streams: []
+  full_refresh:
+    - config_path: "secrets/valid_config.json"
+      configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file
+#  incremental: # TODO uncomment this once you implement incremental sync
+#    - config_path: "secrets/config.json"
+#      configured_catalog_path: "integration_tests/configured_catalog.json"
+#      future_state_path: "integration_tests/abnormal_state.json"

airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+
+# Build latest connector image
+docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-)
+
+# Pull latest acctest image
+docker pull airbyte/source-acceptance-test:latest
+
+# Run
+docker run --rm -it \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp:/tmp \
+    -v $(pwd):/test_input \
+    airbyte/source-acceptance-test \
+    --acceptance-test-config /test_input
+

airbyte-integrations/connector-templates/source-generic/build.gradle

@@ -1,17 +1,6 @@
 plugins {
     // Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image
-    // via ./gradlew :airbyte-integrations:connectors:source-{{snakeCase name}}:build
+    // via ./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build
     id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
-}
-
-airbyteStandardSourceTestFile {
-    // All these input paths must live inside this connector's directory (or subdirectories)
-    configPath = "<path to config file to be used in standard tests>" // TODO
-    configuredCatalogPath = "<path to configured catalog to be used in tests>" // TODO
-    specPath = "<path to spec for use in tests>" // TODO
-}
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
+    id 'airbyte-source-acceptance-test'
 }

buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle

@@ -35,6 +35,11 @@ class AirbyteSourceAcceptanceTestPlugin implements Plugin<Project> {
             project.sourceAcceptanceTest.dependsOn(project.airbyteDockerTest)
         }
 
+        // make sure we create the integrationTest task once
+        if (!project.tasks.findByName('integrationTest')) {
+            project.task('integrationTest')
+        }
+
         project.integrationTest.dependsOn(project.sourceAcceptanceTest)
     }
 }

docs/connector-development/tutorials/build-a-connector-the-hard-way.md

@@ -24,7 +24,7 @@ To run this tutorial, you'll need:
 * Docker, Python, and Java with the versions listed in the [tech stack section](../../understanding-airbyte/tech-stack.md).
 * The `requests` Python package installed via `pip install requests` \(or `pip3` if `pip` is linked to a Python2 installation on your system\)
 
-**A note on running Python**: all the commands below assume that `python` points to a version of Python 3.7. Verify this by running
+**A note on running Python**: all the commands below assume that `python` points to a version of Python 3.7 or greater. Verify this by running
 
 ```bash
 $ python --version
@@ -88,7 +88,7 @@ Head to the connector directory and we should see the following files have been
 ```bash
 $ cd ../../connectors/source-stock-ticker-api
 $ ls
-Dockerfile              NEW_SOURCE_CHECKLIST.md              build.gradle
+Dockerfile                 README.md                  acceptance-test-config.yml acceptance-test-docker.sh  build.gradle
 ```
 
 We'll use each of these files later. But first, let's write some code!
@@ -153,9 +153,10 @@ We'll save this file in the root directory of our connector. Now we have the fol
 ```bash
 $ ls -1
 Dockerfile
-NEW_SOURCE_CHECKLIST.md
+README.md
+acceptance-test-config.yml
+acceptance-test-docker.sh
 build.gradle
-source.py
 spec.json
 ```
 
@@ -494,31 +495,33 @@ First, let's create a configured catalog `fullrefresh_configured_catalog.json` t
 
 ```javascript
 {
-  "streams": [
-    {
-      "stream": {
-        "name": "stock_prices",
-        "supported_sync_modes": [
-          "full_refresh"
-        ],
-        "json_schema": {
-          "properties": {
-            "date": {
-              "type": "string"
-            },
-            "price": {
-              "type": "number"
+    "streams": [
+        {
+            "stream": {
+                "name": "stock_prices",
+                "supported_sync_modes": [
+                    "full_refresh"
+                ],
+                "json_schema": {
+                    "properties": {
+                        "date": {
+                            "type": "string"
+                        },
+                        "price": {
+                            "type": "number"
+                        },
+                        "stock_ticker": {
+                            "type": "string"
+                        }
+                    }
+                }
             },
-            "stock_ticker": {
-              "type": "string"
-            }
-          }
+            "sync_mode": "full_refresh",
+            "destination_sync_mode": "overwrite"
         }
-      },
-      "sync_mode": "full_refresh"
-    }
-  ]
+    ]
 }
+
 ```
 
 Then we'll define the `read` method in `source.py`:
@@ -933,56 +936,61 @@ $ docker run -v $(pwd)/secrets/valid_config.json:/data/config.json -v $(pwd)/ful
 {'type': 'RECORD', 'record': {'stream': 'stock_prices', 'data': {'date': '2020-12-21', 'stock_ticker': 'TSLA', 'price': 649.86}, 'emitted_at': 1608628424000}}
 ```
 
-and with that, we've packaged our connector in a functioning Docker image. The last requirement before calling this connector finished is to pass the [Airbyte Standard Test suite](../testing-connectors/source-acceptance-tests-reference.md).
+and with that, we've packaged our connector in a functioning Docker image. The last requirement before calling this connector finished is to pass the [Airbyte Source Acceptance Test suite](../testing-connectors/source-acceptance-tests-reference.md).
 
 ### 4. Test the connector
 
-The minimum requirement for testing our connector is to pass the Airbyte Standard Test suite. You're encouraged to add custom test cases for your connector where it makes sense to do so e.g: to test edge cases that are not covered by the standard suite. But at the very least, you must pass Airbyte's Standard Test suite.
-
-To integrate with the standard test suite, modify the generated `build.gradle` file as follows:
-
-```groovy
-plugins {
-    // Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image
-    // via ./gradlew :airbyte-integrations:connectors:source-stock-ticker-api:build
-    id 'airbyte-docker'
-    id 'airbyte-standard-source-test-file'
-}
-
-airbyteStandardSourceTestFile {
-    // All these input paths must live inside this connector's directory (or subdirectories)
-    configPath = "secrets/valid_config.json"
-    configuredCatalogPath = "fullrefresh_configured_catalog.json"
-    specPath = "spec.json"
-}
-
-dependencies {
-    implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
-}
+The minimum requirement for testing your connector is to pass the [Source Acceptance Test (SAT)](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) suite. SAT is a blackbox test suite containing a number of tests that validate your connector behaves as intended by the Airbyte Specification. You're encouraged to add custom test cases for your connector where it makes sense to do so e.g: to test edge cases that are not covered by the standard suite. But at the very least, you must pass Airbyte's SATs suite.
+
+The code generator should have already generated a YAML file which configures the test suite. In order to run it, modify the `acceptance-test-config.yaml` file to look like this: 
+
+
+```yaml
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-stock-ticker-api:dev
+tests:
+  spec:
+    - spec_path: "spec.json"
+      config_path: "secrets/valid_config.json"
+  connection:
+    - config_path: "secrets/valid_config.json"
+      status: "succeed"
+    - config_path: "secrets/invalid_config.json"
+      status: "failed"
+  discovery:
+    - config_path: "secrets/valid_config.json"
+  basic_read:
+    - config_path: "secrets/valid_config.json"
+      configured_catalog_path: "fullrefresh_configured_catalog.json"
+      empty_streams: []
+  full_refresh:
+    - config_path: "secrets/valid_config.json"
+      configured_catalog_path: "fullrefresh_configured_catalog.json"
+#  incremental: # TODO uncomment this once you implement incremental sync in part 2 of the tutorial
+#    - config_path: "secrets/config.json"
+#      configured_catalog_path: "integration_tests/configured_catalog.json"
+#      future_state_path: "integration_tests/abnormal_state.json"
 ```
 
-Then **from the Airbyte repository root**, run:
+Then from the connector module directory run
 
 ```bash
-./gradlew clean :airbyte-integrations:connectors:source-stock-ticker-api:integrationTest
+./acceptance-test-docker.sh
 ```
 
 After tests have run, you should see a test summary like:
 
 ```text
-Test run finished after 5049 ms
-[         2 containers found      ]
-[         0 containers skipped    ]
-[         2 containers started    ]
-[         0 containers aborted    ]
-[         2 containers successful ]
-[         0 containers failed     ]
-[         7 tests found           ]
-[         0 tests skipped         ]
-[         7 tests started         ]
-[         0 tests aborted         ]
-[         7 tests successful      ]
-[         0 tests failed          ]
+collecting ...
+ test_core.py ✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓                                                                                                                                                                                                                                                                                                                           95% █████████▌
+ test_full_refresh.py ✓                                                                                                                                                                                                                                                                                                                                    100% ██████████
+
+================== short test summary info ================== 
+SKIPPED [1] source_acceptance_test/plugin.py:56: Skipping TestIncremental.test_two_sequential_reads because not found in the config
+
+Results (8.91s):
+      20 passed
 ```
 
 That's it! We've created a fully functioning connector. Now let's get to the exciting part: using it from the Airbyte UI.
@@ -1034,12 +1042,6 @@ If the Airbyte server isn't already running, start it by running **from the Airb
 docker-compose up
 ```
 
->**_NOTE:_** if you are on an M1 Mac, you have to build Airbyte first and then run it in `dev` mode:
->```bash
->SUB_BUILD=PLATFORM ./gradlew build
->VERSION=dev docker-compose up
->```
-
 When Airbyte server is done starting up, it prints the following banner in the log output \(it can take 10-20 seconds for the server to start\):
 
 ```bash