Source Greenhouse: support incremental syncs (airbytehq#16338)

* airbytehq#1386 Source Greenhouse: support incremental syncs - first try * airbytehq#1386 Source Greenhouse: implement incremental syncs * airbytehq#1386 source greenhouse: upd changelog * Increased unittest to 90 * Updated link in spec * auto-bump connector version [ci skip] * Updated release stage Co-authored-by: Serhii Lazebnyi <serhii.lazebnyi@globallogic.com> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
jhammarstedt · Oct 31, 2022 · 85c4ea5 · 85c4ea5
1 parent efb6730
commit 85c4ea5
Show file tree

Hide file tree

Showing 16 changed files with 645 additions and 346 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -403,11 +403,11 @@
 - name: Greenhouse
   sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44
   dockerRepository: airbyte/source-greenhouse
-  dockerImageTag: 0.2.9
+  dockerImageTag: 0.2.10
   documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse
   icon: greenhouse.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: Harness
   sourceDefinitionId: 6fe89830-d04d-401b-aad6-6552ffa5c4af
   dockerRepository: farosai/airbyte-harness-source

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3661,9 +3661,9 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-greenhouse:0.2.9"
+- dockerImage: "airbyte/source-greenhouse:0.2.10"
   spec:
-    documentationUrl: "https://docs.airbyte.io/integrations/sources/greenhouse"
+    documentationUrl: "https://docs.airbyte.com/integrations/sources/greenhouse"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "Greenhouse Spec"

diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile
@@ -4,13 +4,13 @@ FROM python:3.9-slim
 RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /airbyte/integration_code
-COPY source_greenhouse ./source_greenhouse
-COPY main.py ./
 COPY setup.py ./
 RUN pip install .
+COPY source_greenhouse ./source_greenhouse
+COPY main.py ./
 
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.9
+LABEL io.airbyte.version=0.2.10
 LABEL io.airbyte.name=airbyte/source-greenhouse
diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml
@@ -19,13 +19,14 @@ tests:
       configured_catalog_path: "integration_tests/configured_catalog.json"
       expect_records:
         path: "integration_tests/expected_records.txt"
-        extra_fields: yes
-        exact_order: yes
-        extra_records: no
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
   full_refresh:
     - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog_const_records.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
     - config_path: "secrets/config_users_only.json"
       configured_catalog_path: "integration_tests/configured_catalog_users_only.json"
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "integration_tests/incremental_configured_catalog.json"
+      future_state_path: "integration_tests/abnormal_state.json"
diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json
@@ -0,0 +1,58 @@
+{
+  "candidates": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "demographics_answers": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "users": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "scorecards": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "offers": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "job_stages": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "job_posts": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "interviews": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "jobs": {
+    "updated_at": "2222-01-21T00:00:00.000Z"
+  },
+  "applications": {
+    "applied_at": "2222-01-21T00:00:00.000Z"
+  },
+  "jobs_stages": {
+    "4177046003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "4177048003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "4446240003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "4466310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
+  },
+  "applications_demographics_answers": {
+    "19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
+  },
+  "applications_interviews": {
+    "19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"},
+    "48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"}
+  }
+}