Add SubscriptionUsage stream to Orb Source (#21951)

* wip * mvp * big rework to make incremental possibly work * incremental might be working * add external_customer_id to subscriptions stream * keep state indexed by subscription_id * add a source defined primary key * move super call after assignments * start work on integration tests * format and more test setup * tests and cleanup * fix yield from * format fixes * fix type of new subscriptions property * add subscription with actual usage for integration test config * fix start date parsing and use correct keys in integration test * update docs * bump version from 0.1.4 to 0.1.5 and update changelog * make start_date a required field and set version 0.2.0 instead of 0.1.5 * major version bump from 0.1.4 to 1.0.0 to reflect backwards-incompatible change making start_date a required field * pass start_date in unit test * add sample start_date * start_date already converted * add start_date to sample_config * remove accidental char * auto-bump connector version --------- Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Feb 7, 2023 · 9709adf · 9709adf
1 parent 3297e9e
commit 9709adf
Show file tree

Hide file tree

Showing 16 changed files with 678 additions and 16 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -1216,7 +1216,7 @@
 - name: Orb
   sourceDefinitionId: 7f0455fb-4518-4ec0-b7a3-d808bf8081cc
   dockerRepository: airbyte/source-orb
-  dockerImageTag: 0.1.4
+  dockerImageTag: 1.0.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/orb
   icon: orb.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -10600,7 +10600,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-orb:0.1.4"
+- dockerImage: "airbyte/source-orb:1.0.0"
   spec:
     documentationUrl: "https://docs.withorb.com/"
     connectionSpecification:
@@ -10609,6 +10609,7 @@
       type: "object"
       required:
       - "api_key"
+      - "start_date"
       additionalProperties: true
       properties:
         api_key:
@@ -10622,7 +10623,8 @@
           title: "Start Date"
           pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
           description: "UTC date and time in the format 2022-03-01T00:00:00Z. Any\
-            \ data with created_at before this data will not be synced."
+            \ data with created_at before this data will not be synced. For Subscription\
+            \ Usage, this becomes the `timeframe_start` API parameter."
           examples:
           - "2022-03-01T00:00:00Z"
           order: 2
@@ -10651,6 +10653,15 @@
           description: "Property key names to extract from all events, in order to\
             \ enrich ledger entries corresponding to an event deduction."
           order: 5
+        subscription_usage_grouping_key:
+          type: "string"
+          title: "Subscription usage grouping key (string value)"
+          description: "Property key name to group subscription usage by."
+        plan_id:
+          type: "string"
+          title: "Orb Plan ID for Subscription Usage (string value)"
+          description: "Orb Plan ID to filter subscriptions that should have usage\
+            \ fetched."
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

diff --git a/airbyte-integrations/connectors/source-orb/Dockerfile b/airbyte-integrations/connectors/source-orb/Dockerfile
@@ -34,5 +34,5 @@ COPY source_orb ./source_orb
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-orb
diff --git a/airbyte-integrations/connectors/source-orb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-orb/acceptance-test-config.yml
@@ -4,6 +4,8 @@ connector_image: airbyte/source-orb:dev
 tests:
   spec:
     - spec_path: "source_orb/spec.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.4"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
@@ -23,6 +25,10 @@ tests:
         # This points to a specific customer's credit ledger entries in the state,
         # and this customer is in the integration test account.
         credits_ledger_entries: ["hHQF5BT5jtyj9r7V", "created_at"]
+
+        # This points to a specific subscription's usage entries in the state,
+        # and this subscription is in the integration test account.
+        subscription_usage: ["FDWRvxuBUiFfZech", "timeframe_start"]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
diff --git a/airbyte-integrations/connectors/source-orb/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-orb/integration_tests/abnormal_state.json
@@ -12,5 +12,10 @@
     "hHQF5BT5jtyj9r7V": {
       "created_at": "2122-01-01T00:00:00Z"
     }
+  },
+  "subscription_usage": {
+    "FDWRvxuBUiFfZech": {
+      "timeframe_start": "2122-01-01T00:00:00Z"
+    }
   }
 }
diff --git a/airbyte-integrations/connectors/source-orb/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-orb/integration_tests/configured_catalog.json
@@ -47,6 +47,22 @@
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "subscription_usage",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["timeframe_start"],
+        "source_defined_primary_key": [
+          ["subscription_id"],
+          ["billable_metric_id"],
+          ["timeframe_start"]
+        ]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
     }
   ]
 }
diff --git a/airbyte-integrations/connectors/source-orb/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-orb/integration_tests/sample_config.json
@@ -1,3 +1,4 @@
 {
-  "api_key": "<sample_api_key>"
+  "api_key": "<sample_api_key>",
+  "start_date": "2023-01-25T00:00:00Z"
 }
diff --git a/airbyte-integrations/connectors/source-orb/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-orb/integration_tests/sample_state.json
@@ -12,5 +12,10 @@
     "7c507794-7413-4467-8f1d-d3785a6c65ca": {
       "created_at": "2022-01-01T00:00:00Z"
     }
+  },
+  "subscription_usage": {
+    "someId": {
+      "timeframe_start": "2022-01-01T00:00:00Z"
+    }
   }
 }
diff --git a/airbyte-integrations/connectors/source-orb/source_orb/schemas/subscription_usage.json b/airbyte-integrations/connectors/source-orb/source_orb/schemas/subscription_usage.json
@@ -0,0 +1,34 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": ["null", "object"],
+  "properties": {
+    "quantity": {
+      "type": "number"
+    },
+    "timeframe_start": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "timeframe_end": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "billable_metric_name": {
+      "type": "string"
+    },
+    "billable_metric_id": {
+      "type": "string"
+    },
+    "subscription_id": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "quantity",
+    "timeframe_start",
+    "timeframe_end",
+    "billable_metric_name",
+    "billable_metric_id",
+    "subscription_id"
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-orb/source_orb/schemas/subscriptions.json b/airbyte-integrations/connectors/source-orb/source_orb/schemas/subscriptions.json
@@ -12,6 +12,9 @@
     "customer_id": {
       "type": "string"
     },
+    "external_customer_id": {
+      "type": ["null", "string"]
+    },
     "start_date": {
       "type": ["null", "string"],
       "format": "date-time"