From e2100c48eca6e2918abc49d1796977efb62628c6 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Thu, 9 Feb 2023 11:23:00 +0200 Subject: [PATCH 1/6] Source Hubspot: turn on default HttpAvailabilityStrategy (#22479) Signed-off-by: Sergey Chvalyuk --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-hubspot/Dockerfile | 2 +- .../integration_tests/expected_records.jsonl | 4 +- .../source-hubspot/source_hubspot/source.py | 55 +---------------- .../source-hubspot/source_hubspot/streams.py | 12 ---- .../source-hubspot/unit_tests/conftest.py | 5 -- .../source-hubspot/unit_tests/test_source.py | 59 +++++++------------ docs/integrations/sources/hubspot.md | 5 +- 9 files changed, 29 insertions(+), 117 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 42d71e5909a1..6e358a7cc6cd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -799,7 +799,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.3.1 + dockerImageTag: 0.3.2 documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e3a0db465242..d7fa8ecc7bb2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6521,7 +6521,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.3.1" +- dockerImage: "airbyte/source-hubspot:0.3.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/hubspot" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 634c0a82b7ad..8b26009b81d9 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.2 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl index 8e0d8027d709..655fca74b102 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl @@ -21,7 +21,7 @@ {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 40032127, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 321690519}, "name": "Unnamed workflow - Mon Mar 15 2021 12:58:03 GMT+0200 (cloned)", "id": 40032127, "type": "DRIP_DELAY", "portalId": 8727216, "insertedAt": 1675124258190, "updatedAt": 1675124308226, "enabled": true, "description": "", "contactListIds": {"enrolled": 167, "active": 168, "completed": 169, "succeeded": 170}, "creationSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "createdByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "clonedFromWorkflowId": 23314874, "createdAt": 1675124258186}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "updatedByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "updatedAt": 1675124308226}, "originalAuthorUserId": 12282590, "personaTagIds": [], "lastUpdatedByUserId": 12282590, "contactCounts": {"active": 0, "enrolled": 0}}, "emitted_at": 1675125115992} {"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0.0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1.0, "hs_object_id": 4992593519, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": true, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0.0, "num_associated_deals": 1.0, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200.0, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false}, "emitted_at": 1675336936136} {"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0.0, "hs_num_child_companies": 0.0, "hs_num_contacts_with_buying_roles": 0.0, "hs_num_decision_makers": 0.0, "hs_num_open_deals": 0.0, "hs_object_id": 5000787595, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": true, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0.0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50.0, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false}, "emitted_at": 1675336936137} -{"stream": "companies", "data": {"id": "11481383026", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": 1.0, "city": "Test", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": null, "createdate": "2023-01-30T23:22:56.969000+00:00", "days_to_close": null, "description": "Test", "domain": "test.test", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": null, "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-01-30T23:22:56.969000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": "2023-02-03T07:00:00+00:00", "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-30T23:51:53.295000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0.0, "hs_num_child_companies": 0.0, "hs_num_contacts_with_buying_roles": 0.0, "hs_num_decision_makers": 0.0, "hs_num_open_deals": 0.0, "hs_object_id": 11481383026, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:22:56.969000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": "ACCOUNTING", "is_public": null, "lifecyclestage": "lead", "linkedin_company_page": "Test", "linkedinbio": null, "name": "Test", "notes_last_contacted": null, "notes_last_updated": "2023-01-30T23:51:47.542000+00:00", "notes_next_activity_date": "2023-02-03T07:00:00+00:00", "num_associated_contacts": 0.0, "num_associated_deals": null, "num_contacted_notes": 0.0, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": 2.0, "numberofemployees": 1.0, "phone": null, "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "Test", "timezone": "3", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": null, "type": "PROSPECT", "web_technologies": null, "website": "test.test", "zip": "00000"}, "createdAt": "2023-01-30T23:22:56.969Z", "updatedAt": "2023-01-30T23:51:53.295Z", "archived": false}, "emitted_at": 1675336936473} +{"stream": "companies", "data": {"id": "11481383026", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": 1.0, "city": "Test", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": null, "createdate": "2023-01-30T23:22:56.969000+00:00", "days_to_close": null, "description": "Test", "domain": "test.test", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": null, "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-01-30T23:22:56.969000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": "2023-02-03T07:00:00+00:00", "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-02-03T07:03:57.136000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0.0, "hs_num_child_companies": 0.0, "hs_num_contacts_with_buying_roles": 0.0, "hs_num_decision_makers": 0.0, "hs_num_open_deals": 0.0, "hs_object_id": 11481383026, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:22:56.969000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": "ACCOUNTING", "is_public": null, "lifecyclestage": "lead", "linkedin_company_page": "Test", "linkedinbio": null, "name": "Test", "notes_last_contacted": null, "notes_last_updated": "2023-02-03T07:00:00+00:00", "notes_next_activity_date": null, "num_associated_contacts": 0.0, "num_associated_deals": null, "num_contacted_notes": 0.0, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": 2.0, "numberofemployees": 1.0, "phone": null, "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "Test", "timezone": "3", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": null, "type": "PROSPECT", "web_technologies": null, "website": "test.test", "zip": "00000"}, "createdAt": "2023-01-30T23:22:56.969Z", "updatedAt": "2023-02-03T07:03:57.136Z", "archived": false}, "emitted_at": 1675336936473} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1675124235515, "vid": 2501, "is-member": true}, "emitted_at": 1675337054631} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 61, "internal-list-id": 2147483643, "timestamp": 1675124259228, "vid": 2501, "is-member": true}, "emitted_at": 1675337054632} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 166, "internal-list-id": 2147483643, "timestamp": 1675120848102, "vid": 2501, "is-member": true}, "emitted_at": 1675337054632} @@ -39,5 +39,5 @@ {"stream": "deals", "data": {"id": "5313728065", "properties": {"amount": null, "amount_in_home_currency": null, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2021-05-31T10:17:06.028000+00:00", "createdate": "2021-05-21T10:17:06.028000+00:00", "days_to_close": 10.0, "dealname": "Michael Scott - New Deal", "dealstage": "appointmentscheduled", "dealtype": null, "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_2": "", "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0.0, "hs_closed_amount_in_home_currency": 0.0, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-05-21T10:17:28.365000+00:00", "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-05-21T10:17:28.365000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": 0.2, "hs_exchange_rate": null, "hs_forecast_amount": null, "hs_forecast_probability": null, "hs_is_closed": true, "hs_is_closed_won": true, "hs_is_deal_split": true, "hs_lastmodifieddate": "2023-01-30T23:10:04.054000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": 0.0, "hs_num_target_accounts": 0.0, "hs_object_id": 5313728065, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": null, "hs_projected_amount_in_home_currency": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2021-05-21T10:17:28.365000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0.0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-05-21T10:17:06.028Z", "updatedAt": "2023-01-30T23:10:04.054Z", "archived": false, "companies": ["4992593519", "4992593519"]}, "emitted_at": 1675337233906} {"stream": "deals", "data": {"id": "5388213824", "properties": {"amount": 10.0, "amount_in_home_currency": 10.0, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-06-02T14:11:49.985000+00:00", "days_to_close": 0.0, "dealname": "Tim's Newer Deal", "dealstage": null, "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "65568071", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0.0, "hs_closed_amount_in_home_currency": 0.0, "hs_created_by_user_id": null, "hs_createdate": "2021-06-02T14:11:49.985000+00:00", "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": null, "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.0, "hs_deal_stage_probability_shadow": 0.0, "hs_exchange_rate": null, "hs_forecast_amount": 10.0, "hs_forecast_probability": null, "hs_is_closed": null, "hs_is_closed_won": true, "hs_is_deal_split": true, "hs_lastmodifieddate": "2023-01-30T23:10:05.702000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": 0.0, "hs_num_target_accounts": 0.0, "hs_object_id": 5388213824, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 0.0, "hs_projected_amount_in_home_currency": 0.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 10.0, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "23660227", "hubspot_owner_assigneddate": "2021-06-02T14:11:49.985000+00:00", "hubspot_owner_id": "65568071", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0.0, "num_contacted_notes": null, "num_notes": null, "pipeline": null}, "createdAt": "2021-06-02T14:11:49.985Z", "updatedAt": "2023-01-30T23:10:05.702Z", "archived": false, "companies": ["5000526215", "5000526215"], "line items": ["2089616136"]}, "emitted_at": 1675337233907} {"stream": "deals", "data": {"id": "5388306989", "properties": {"amount": 60000.0, "amount_in_home_currency": 60000.0, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-06-02T14:12:00.029000+00:00", "days_to_close": 0.0, "dealname": "Tim's Newer Deal 2", "dealstage": null, "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "65568071", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0.0, "hs_closed_amount_in_home_currency": 0.0, "hs_created_by_user_id": null, "hs_createdate": "2021-06-02T14:12:00.029000+00:00", "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": null, "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.0, "hs_deal_stage_probability_shadow": 0.0, "hs_exchange_rate": null, "hs_forecast_amount": 60000.0, "hs_forecast_probability": null, "hs_is_closed": null, "hs_is_closed_won": true, "hs_is_deal_split": true, "hs_lastmodifieddate": "2023-01-30T23:10:04.959000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": 0.0, "hs_num_target_accounts": 0.0, "hs_object_id": 5388306989, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 0.0, "hs_projected_amount_in_home_currency": 0.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "23660227", "hubspot_owner_assigneddate": "2021-06-02T14:12:00.029000+00:00", "hubspot_owner_id": "65568071", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0.0, "num_contacted_notes": null, "num_notes": null, "pipeline": null}, "createdAt": "2021-06-02T14:12:00.029Z", "updatedAt": "2023-01-30T23:10:04.959Z", "archived": false, "companies": ["5000526215", "5000526215"]}, "emitted_at": 1675337233907} -{"stream": "deals", "data": {"id": "11936210032", "properties": {"amount": 34.0, "amount_in_home_currency": 34.0, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2023-01-30T23:41:04.079000+00:00", "createdate": "2023-01-30T23:41:12.865000+00:00", "days_to_close": 0.0, "dealname": "test", "dealstage": "appointmentscheduled", "dealtype": null, "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 34.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "OFFLINE", "hs_analytics_latest_source_data_1": "CRM_UI", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "CRM_UI", "hs_analytics_latest_source_data_2": "userId:12282590", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "userId:12282590", "hs_analytics_latest_source_timestamp": "2023-01-30T23:17:10.053000+00:00", "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": "2023-01-30T23:17:10.053000+00:00", "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0.0, "hs_closed_amount_in_home_currency": 0.0, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:12.865000+00:00", "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2023-01-30T23:41:12.865000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": 0.2, "hs_exchange_rate": null, "hs_forecast_amount": 34.0, "hs_forecast_probability": null, "hs_is_closed": true, "hs_is_closed_won": true, "hs_is_deal_split": true, "hs_lastmodifieddate": "2023-01-31T00:31:37.738000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": 0.0, "hs_num_target_accounts": 0.0, "hs_object_id": 11936210032, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 6.800000000000001, "hs_projected_amount_in_home_currency": 6.800000000000001, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 34.0, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:41:12.865000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": "2023-01-30T23:41:23.198000+00:00", "notes_next_activity_date": "2023-02-03T07:00:00+00:00", "num_associated_contacts": 1.0, "num_contacted_notes": 0.0, "num_notes": 2.0, "pipeline": "default"}, "createdAt": "2023-01-30T23:41:12.865Z", "updatedAt": "2023-01-31T00:31:37.738Z", "archived": false, "line items": ["4617726725"], "contacts": ["2501"]}, "emitted_at": 1675346744980} +{"stream": "deals", "data": {"id": "11936210032", "properties": {"amount": 34.0, "amount_in_home_currency": 34.0, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2023-01-30T23:41:04.079000+00:00", "createdate": "2023-01-30T23:41:12.865000+00:00", "days_to_close": 0.0, "dealname": "test", "dealstage": "appointmentscheduled", "dealtype": null, "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 34.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "OFFLINE", "hs_analytics_latest_source_data_1": "CRM_UI", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "CRM_UI", "hs_analytics_latest_source_data_2": "userId:12282590", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "userId:12282590", "hs_analytics_latest_source_timestamp": "2023-01-30T23:17:10.053000+00:00", "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": "2023-01-30T23:17:10.053000+00:00", "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0.0, "hs_closed_amount_in_home_currency": 0.0, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:12.865000+00:00", "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2023-01-30T23:41:12.865000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": 0.2, "hs_exchange_rate": null, "hs_forecast_amount": 34.0, "hs_forecast_probability": null, "hs_is_closed": true, "hs_is_closed_won": true, "hs_is_deal_split": true, "hs_lastmodifieddate": "2023-02-03T07:06:10.268000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": 0.0, "hs_num_target_accounts": 0.0, "hs_object_id": 11936210032, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 6.800000000000001, "hs_projected_amount_in_home_currency": 6.800000000000001, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 34.0, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:41:12.865000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": "2023-02-03T07:00:00+00:00", "notes_next_activity_date": null, "num_associated_contacts": 1.0, "num_contacted_notes": 0.0, "num_notes": 2.0, "pipeline": "default"}, "createdAt": "2023-01-30T23:41:12.865Z", "updatedAt": "2023-02-03T07:06:10.268Z", "archived": false, "line items": ["4617726725"], "contacts": ["2501"]}, "emitted_at": 1675346744980} {"stream": "tickets", "data": {"id": "1401690016", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2023-01-30T23:52:42.464000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2023-01-30T23:52:42.464000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2023-01-30T23:52:43.939000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0.0, "hs_num_times_contacted": 0.0, "hs_object_id": 1401690016, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 1401690016, "hs_ticket_priority": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:52:42.464000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": null, "subject": "test", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2023-01-30T23:52:42.464Z", "updatedAt": "2023-01-30T23:52:43.939Z", "archived": false}, "emitted_at": 1675337293708} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 3674d3fc9bae..e994ee321563 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -4,17 +4,12 @@ import logging from itertools import chain -from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union +from typing import Any, List, Mapping, Optional, Tuple import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.utils.schema_helpers import split_config -from airbyte_cdk.utils.event_timing import create_timer -from airbyte_cdk.utils.traced_exception import AirbyteTracedException from requests import HTTPError from source_hubspot.streams import ( API, @@ -136,51 +131,3 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: available_streams = streams return available_streams - - def read( - self, - logger: logging.Logger, - config: Mapping[str, Any], - catalog: ConfiguredAirbyteCatalog, - state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, - ) -> Iterator[AirbyteMessage]: - """ - This method is overridden to check whether the stream `quotes` exists in the source, if not skip reading that stream. - """ - logger.info(f"Starting syncing {self.name}") - config, internal_config = split_config(config) - # TODO assert all streams exist in the connector - # get the streams once in case the connector needs to make any queries to generate them - stream_instances = {s.name: s for s in self.streams(config)} - state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state) - self._stream_to_instance_map = stream_instances - with create_timer(self.name) as timer: - for configured_stream in catalog.streams: - stream_instance = stream_instances.get(configured_stream.stream.name) - if not stream_instance and configured_stream.stream.name == "quotes": - logger.warning("Stream `quotes` does not exist in the source. Skip reading `quotes` stream.") - continue - if not stream_instance: - raise KeyError( - f"The requested stream {configured_stream.stream.name} was not found in the source. Available streams: {stream_instances.keys()}" - ) - - try: - yield from self._read_stream( - logger=logger, - stream_instance=stream_instance, - configured_stream=configured_stream, - state_manager=state_manager, - internal_config=internal_config, - ) - except Exception as e: - logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}") - display_message = stream_instance.get_error_display_message(e) - if display_message: - raise AirbyteTracedException.from_exception(e, message=display_message) from e - raise e - finally: - logger.info(f"Finished syncing {self.name}") - logger.info(timer.report()) - - logger.info(f"Finished syncing {self.name}") diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index b724fac78d9d..ebd24f1c6bcb 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -15,7 +15,6 @@ import requests from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer @@ -206,14 +205,9 @@ class Stream(HttpStream, ABC): primary_key = None filter_old_records: bool = True denormalize_records: bool = False # one record from API response can result in multiple records emitted - raise_on_http_errors: bool = True granted_scopes: Set = None properties_scopes: Set = None - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - @property @abstractmethod def scopes(self) -> Set[str]: @@ -263,12 +257,6 @@ def __init__(self, api: API, start_date: Union[str, pendulum.datetime], credenti if creds_title in (OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS): self._authenticator = api.get_authenticator() - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == HTTPStatus.FORBIDDEN: - setattr(self, "raise_on_http_errors", False) - logger.warning("You have not permission to API for this stream. " "Please check your scopes for Hubspot account.") - return super().should_retry(response) - def backoff_time(self, response: requests.Response) -> Optional[float]: if response.status_code == codes.too_many_requests: return float(response.headers.get("Retry-After", 3)) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index d05ec339c6eb..3e3eacc9bd3a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -42,11 +42,6 @@ def some_credentials_fixture(): return {"credentials_title": "Private App Credentials", "access_token": "wrong token"} -@pytest.fixture(name="creds_with_wrong_permissions") -def creds_with_wrong_permissions(): - return {"credentials_title": "Private App Credentials", "access_token": "THIS-IS-THE-ACCESS_TOKEN"} - - @pytest.fixture(name="fake_properties_list") def fake_properties_list(): return [f"property_number_{i}" for i in range(NUMBER_OF_PROPERTIES)] diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index f63103e0b99c..0de88309967c 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -13,7 +13,7 @@ from source_hubspot.errors import HubspotRateLimited from source_hubspot.helpers import APIv3Property from source_hubspot.source import SourceHubspot -from source_hubspot.streams import API, Companies, Deals, Engagements, Products, Stream, Workflows +from source_hubspot.streams import API, Companies, Deals, Engagements, Products, Stream from .utils import read_full_refresh, read_incremental @@ -134,38 +134,30 @@ def test_check_connection_backoff_on_server_error(requests_mock, config): assert not error -def test_wrong_permissions_api_key(requests_mock, creds_with_wrong_permissions, common_params, caplog): - """ - Error with API Key Permissions to particular stream, - typically this issue raises along with calling `workflows` stream with API Key - that doesn't have required permissions to read the stream. - """ - - # Mapping tipical response for mocker +def test_stream_forbidden(requests_mock, config, caplog): json = { "status": "error", - "message": f'This hapikey ({creds_with_wrong_permissions.get("api_key")}) does not have proper permissions! (requires any of [automation-access])', - "correlationId": "2fe0a9af-3609-45c9-a4d7-83a1774121aa", - } - - # We expect something like this - expected_warining_message = { - "type": "LOG", - "log": { - "level": "WARN", - "message": f'Stream `workflows` cannot be procced. This hapikey ({creds_with_wrong_permissions.get("api_key")}) does not have proper permissions! (requires any of [automation-access])', - }, + "message": "This access_token does not have proper permissions!", } + requests_mock.get("https://api.hubapi.com/automation/v3/workflows", json=json, status_code=403) - # Create test_stream instance - test_stream = Workflows(**common_params) - - # Mocking Request - requests_mock.register_uri("GET", test_stream.url, json=json, status_code=403) - records = list(test_stream.read_records(sync_mode=SyncMode.full_refresh)) + catalog = ConfiguredAirbyteCatalog.parse_obj({ + "streams": [ + { + "stream": { + "name": "workflows", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] + }) - # match logged expected logged warning message with output given from preudo-output - assert expected_warining_message["log"]["message"] in caplog.text + records = list(SourceHubspot().read(logger, config, catalog, {})) + assert json["message"] in caplog.text + records = [r for r in records if r.type == Type.RECORD] assert not records @@ -328,17 +320,6 @@ def configured_catalog_fixture(): return ConfiguredAirbyteCatalog.parse_obj(configured_catalog) -def test_it_should_not_read_quotes_stream_if_it_does_not_exist_in_client(oauth_config, configured_catalog): - """ - If 'quotes' stream is not in the client, it should skip it. - """ - source = SourceHubspot() - - all_records = list(source.read(logger, config=oauth_config, catalog=configured_catalog, state=None)) - records = [record for record in all_records if record.type == Type.RECORD] - assert not records - - def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(requests_mock, common_params, fake_properties_list): """ If there are more than 10,000 records that would be returned by the Hubspot search endpoint, diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 0aeac0601c5c..2e6c214da744 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -126,8 +126,9 @@ Now that you have set up the Hubspot source connector, check out the following H | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------| -| 0.3.1 | 2023-01-27 | [22009](https://github.com/airbytehq/airbyte/pull/22009) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.3.0 | 2022-10-27 | [18546](https://github.com/airbytehq/airbyte/pull/18546) | Sunsetting API Key authentication. `Quotes` stream is no longer available | +| 0.3.2 | 2023-02-07 | [22479](https://github.com/airbytehq/airbyte/pull/22479) | Turn on default HttpAvailabilityStrategy | +| 0.3.1 | 2023-01-27 | [22009](https://github.com/airbytehq/airbyte/pull/22009) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.3.0 | 2022-10-27 | [18546](https://github.com/airbytehq/airbyte/pull/18546) | Sunsetting API Key authentication. `Quotes` stream is no longer available | | 0.2.2 | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation | | 0.2.1 | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state. | | 0.2.0 | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API. | From 4f97ad74ef70737ade5a3cbac5a06823e84805df Mon Sep 17 00:00:00 2001 From: danidelvalle Date: Thu, 9 Feb 2023 13:08:22 +0100 Subject: [PATCH 2/6] Docs: set proper env vars for airbyte-username and airbyte-password (#22634) --- octavia-cli/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octavia-cli/README.md b/octavia-cli/README.md index a411f99d32fb..4fde4a293921 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -138,10 +138,10 @@ docker compose run octavia-cli ` ### `octavia` command flags | **Flag** | **Description** | **Env Variable** | **Default** | -| ---------------------------------------- | --------------------------------------------------------------------------------- | -------------------------- | ------------------------------------------------------ | +| ---------------------------------------- | --------------------------------------------------------------------------------- |----------------------------| ------------------------------------------------------ | | `--airbyte-url` | Airbyte instance URL. | `AIRBYTE_URL` | `http://localhost:8000` | -| `--airbyte-username` | Airbyte instance username (basic auth). | `AIRBYTE_URL` | `airbyte` | -| `--airbyte-password` | Airbyte instance password (basic auth). | `AIRBYTE_URL` | `password` | +| `--airbyte-username` | Airbyte instance username (basic auth). | `AIRBYTE_USERNAME` | `airbyte` | +| `--airbyte-password` | Airbyte instance password (basic auth). | `AIRBYTE_PASSWORD` | `password` | | `--workspace-id` | Airbyte workspace id. | `AIRBYTE_WORKSPACE_ID` | The first workspace id found on your Airbyte instance. | | `--enable-telemetry/--disable-telemetry` | Enable or disable the sending of telemetry data. | `OCTAVIA_ENABLE_TELEMETRY` | True | | `--api-http-header` | HTTP Header value pairs passed while calling Airbyte's API | None | None | From fd6497c129a1caaad3bf111f389ed547a55fafa4 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 9 Feb 2023 13:34:21 +0100 Subject: [PATCH 3/6] Create scans of gradle builds (#22583) --- tools/bin/release_version.sh | 4 ++-- tools/bin/release_version_octavia.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/bin/release_version.sh b/tools/bin/release_version.sh index a9c21bc6110d..ccfa0f7c0e63 100755 --- a/tools/bin/release_version.sh +++ b/tools/bin/release_version.sh @@ -29,8 +29,8 @@ docker login -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}" source ./tools/bin/bump_version.sh echo "Building and publishing PLATFORM version $NEW_VERSION for git revision $GIT_REVISION..." -VERSION=$NEW_VERSION SUB_BUILD=PLATFORM ./gradlew clean build -VERSION=$NEW_VERSION SUB_BUILD=PLATFORM ./gradlew publish +VERSION=$NEW_VERSION SUB_BUILD=PLATFORM ./gradlew clean build --scan +VERSION=$NEW_VERSION SUB_BUILD=PLATFORM ./gradlew publish --scan # Container should be running before build starts # It generates binaries to build images for different CPU architecture diff --git a/tools/bin/release_version_octavia.sh b/tools/bin/release_version_octavia.sh index 5bfa746294cb..942aa2009736 100755 --- a/tools/bin/release_version_octavia.sh +++ b/tools/bin/release_version_octavia.sh @@ -19,6 +19,6 @@ docker login --username "${DOCKER_HUB_USERNAME}" --password "${DOCKER_HUB_PASSWO source ./tools/bin/bump_version.sh echo "Building and publishing OCTAVIA version ${NEW_VERSION} for git revision ${GIT_REVISION}..." -VERSION=$NEW_VERSION SUB_BUILD=OCTAVIA_CLI ./gradlew clean build +VERSION=$NEW_VERSION SUB_BUILD=OCTAVIA_CLI ./gradlew clean build --scan ./octavia-cli/publish.sh ${NEW_VERSION} ${GIT_REVISION} echo "Completed building and publishing OCTAVIA..." From b9d4f129b834fa858fc8e86d07286b2895006816 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Thu, 9 Feb 2023 15:31:54 +0200 Subject: [PATCH 4/6] Source Salesforce: handle too many properties (#22597) * #1403 source Salesforce: handle too many properties * #1403 source Salesforce: update changelog * #1403 source salesforce: log warning and skip inconsistent records * #1403 source salesforce: review fixes * auto-bump connector version --------- Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-salesforce/Dockerfile | 2 +- .../source_salesforce/source.py | 29 +-- .../source_salesforce/streams.py | 169 +++++++++++++++--- .../source-salesforce/unit_tests/api_test.py | 82 ++++++++- .../source-salesforce/unit_tests/conftest.py | 21 +++ docs/integrations/sources/salesforce.md | 1 + 8 files changed, 257 insertions(+), 51 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 6e358a7cc6cd..f1f2bcc64b11 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1565,7 +1565,7 @@ - name: Salesforce sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 dockerRepository: airbyte/source-salesforce - dockerImageTag: 2.0.0 + dockerImageTag: 2.0.1 documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce icon: salesforce.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index d7fa8ecc7bb2..5b3d38382892 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -13106,7 +13106,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:2.0.0" +- dockerImage: "airbyte/source-salesforce:2.0.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index a8df6e1fccc1..46b2ca9ce8ee 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.0 +LABEL io.airbyte.version=2.0.1 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py index a2a314ccd1c2..c3e73f4752c4 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py @@ -17,7 +17,7 @@ from requests import codes, exceptions # type: ignore[import] from .api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce -from .streams import BulkIncrementalSalesforceStream, BulkSalesforceStream, Describe, IncrementalSalesforceStream, SalesforceStream +from .streams import BulkIncrementalSalesforceStream, BulkSalesforceStream, Describe, IncrementalRestSalesforceStream, RestSalesforceStream class AirbyteStopSync(AirbyteTracedException): @@ -59,17 +59,10 @@ def _get_api_type(cls, stream_name, properties): properties_not_supported_by_bulk = { key: value for key, value in properties.items() if value.get("format") == "base64" or "object" in value["type"] } - properties_length = len(",".join(p for p in properties)) - rest_required = stream_name in UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS or properties_not_supported_by_bulk - # If we have a lot of properties we can overcome REST API URL length and get an error: "reason: URI Too Long". - # For such cases connector tries to use BULK API because it uses POST request and passes properties in the request body. - bulk_required = properties_length + 2000 > Salesforce.REQUEST_SIZE_LIMITS - - if rest_required and not bulk_required: + if rest_required: return "rest" - if not rest_required: - return "bulk" + return "bulk" @classmethod def generate_streams( @@ -79,6 +72,7 @@ def generate_streams( sf_object: Salesforce, ) -> List[Stream]: """ "Generates a list of stream by their names. It can be used for different tests too""" + logger = logging.getLogger() authenticator = TokenAuthenticator(sf_object.access_token) stream_properties = sf_object.generate_schemas(stream_objects) streams = [] @@ -88,7 +82,7 @@ def generate_streams( api_type = cls._get_api_type(stream_name, selected_properties) if api_type == "rest": - full_refresh, incremental = SalesforceStream, IncrementalSalesforceStream + full_refresh, incremental = RestSalesforceStream, IncrementalRestSalesforceStream elif api_type == "bulk": full_refresh, incremental = BulkSalesforceStream, BulkIncrementalSalesforceStream else: @@ -98,10 +92,17 @@ def generate_streams( pk, replication_key = sf_object.get_pk_and_replication_key(json_schema) streams_kwargs.update(dict(sf_api=sf_object, pk=pk, stream_name=stream_name, schema=json_schema, authenticator=authenticator)) if replication_key and stream_name not in UNSUPPORTED_FILTERING_STREAMS: - streams.append(incremental(**streams_kwargs, replication_key=replication_key, start_date=config.get("start_date"))) + stream = incremental(**streams_kwargs, replication_key=replication_key, start_date=config.get("start_date")) else: - streams.append(full_refresh(**streams_kwargs)) - + stream = full_refresh(**streams_kwargs) + if api_type == "rest" and not stream.primary_key and stream.too_many_properties: + logger.warning( + f"Can not instantiate stream {stream_name}. " + f"It is not supported by the BULK API and can not be implemented via REST because the number of its properties " + f"exceeds the limit and it lacks a primary key." + ) + continue + streams.append(stream) return streams def streams(self, config: Mapping[str, Any]) -> List[Stream]: diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 465a8505303e..7611b23a8cc4 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -9,14 +9,14 @@ import time from abc import ABC from contextlib import closing -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Type, Union +from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Type, Union import pandas as pd import pendulum import requests # type: ignore[import] from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode -from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.core import Stream, StreamData from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from numpy import nan @@ -38,6 +38,7 @@ class SalesforceStream(HttpStream, ABC): page_size = 2000 transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) encoding = DEFAULT_ENCODING + MAX_PROPERTIES_LENGTH = Salesforce.REQUEST_SIZE_LIMITS - 2000 def __init__( self, sf_api: Salesforce, pk: str, stream_name: str, sobject_options: Mapping[str, Any] = None, schema: dict = None, **kwargs @@ -65,6 +66,31 @@ def url_base(self) -> str: def availability_strategy(self) -> Optional["AvailabilityStrategy"]: return None + @property + def too_many_properties(self): + selected_properties = self.get_json_schema().get("properties", {}) + properties_length = len(",".join(p for p in selected_properties)) + return properties_length > self.MAX_PROPERTIES_LENGTH + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json()["records"] + + def get_json_schema(self) -> Mapping[str, Any]: + if not self.schema: + self.schema = self.sf_api.generate_schema(self.name) + return self.schema + + def get_error_display_message(self, exception: BaseException) -> Optional[str]: + if isinstance(exception, exceptions.ConnectionError): + return f"After {self.max_retries} retries the connector has failed with a network error. It looks like Salesforce API experienced temporary instability, please try again later." + return super().get_error_display_message(exception) + + +class RestSalesforceStream(SalesforceStream): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.primary_key or not self.too_many_properties + def path(self, next_page_token: Mapping[str, Any] = None, **kwargs: Any) -> str: if next_page_token: """ @@ -80,7 +106,11 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return {"next_token": next_token} if next_token else None def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + property_chunk: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: """ Salesforce SOQL Query: https://developer.salesforce.com/docs/atlas.en-us.232.0.api_rest.meta/api_rest/dome_queryall.htm @@ -91,21 +121,31 @@ def request_params( """ return {} - selected_properties = self.get_json_schema().get("properties", {}) - query = f"SELECT {','.join(selected_properties.keys())} FROM {self.name} " + property_chunk = property_chunk or {} + query = f"SELECT {','.join(property_chunk.keys())} FROM {self.name} " if self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: query += f"ORDER BY {self.primary_key} ASC" return {"q": query} - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json()["records"] + def chunk_properties(self) -> Iterable[Mapping[str, Any]]: + selected_properties = self.get_json_schema().get("properties", {}) - def get_json_schema(self) -> Mapping[str, Any]: - if not self.schema: - self.schema = self.sf_api.generate_schema(self.name) - return self.schema + summary_length = 0 + local_properties = {} + for property_name, value in selected_properties.items(): + current_property_length = len(property_name) + 1 # properties are split with commas + if current_property_length + summary_length >= self.MAX_PROPERTIES_LENGTH: + yield local_properties + local_properties = {} + summary_length = 0 + + local_properties[property_name] = value + summary_length += current_property_length + + if local_properties: + yield local_properties def read_records( self, @@ -113,10 +153,12 @@ def read_records( cursor_field: List[str] = None, stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: + ) -> Iterable[StreamData]: try: - yield from super().read_records( - sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + yield from self._read_pages( + lambda req, res, state, _slice: self.parse_response(res, stream_slice=_slice, stream_state=state), + stream_slice, + stream_state, ) except exceptions.HTTPError as error: """ @@ -135,10 +177,83 @@ def read_records( return raise error - def get_error_display_message(self, exception: BaseException) -> Optional[str]: - if isinstance(exception, exceptions.ConnectionError): - return f"After {self.max_retries} retries the connector has failed with a network error. It looks like Salesforce API experienced temporary instability, please try again later." - return super().get_error_display_message(exception) + def _read_pages( + self, + records_generator_fn: Callable[ + [requests.PreparedRequest, requests.Response, Mapping[str, Any], Mapping[str, Any]], Iterable[StreamData] + ], + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[StreamData]: + stream_state = stream_state or {} + pagination_complete = False + records = {} + next_pages = {} + + while not pagination_complete: + index = 0 + for index, property_chunk in enumerate(self.chunk_properties()): + request, response = self._fetch_next_page(stream_slice, stream_state, next_pages.get(index), property_chunk) + next_pages[index] = self.next_page_token(response) + chunk_page_records = records_generator_fn(request, response, stream_state, stream_slice) + if not self.too_many_properties: + # this is the case when a stream has no primary key + # (is allowed when properties length does not exceed the maximum value) + # so there would be a single iteration, therefore we may and should yield records immediately + yield from chunk_page_records + break + chunk_page_records = {record[self.primary_key]: record for record in chunk_page_records} + + for record_id, record in chunk_page_records.items(): + if record_id not in records: + records[record_id] = (record, 1) + continue + incomplete_record, counter = records[record_id] + incomplete_record.update(record) + counter += 1 + records[record_id] = (incomplete_record, counter) + + for record_id, (record, counter) in records.items(): + if counter != index + 1: + # Because we make multiple calls to query N records (each call to fetch X properties of all the N records), + # there's a chance that the number of records corresponding to the query may change between the calls. This + # may result in data inconsistency. We skip such records for now and log a warning message. + self.logger.warning( + f"Inconsistent record with primary key {record_id} found. It consists of {counter} chunks instead of {index + 1}. " + f"Skipping it." + ) + continue + yield record + + records = {} + + if not any(next_pages.values()): + pagination_complete = True + + # Always return an empty generator just in case no records were ever yielded + yield from [] + + def _fetch_next_page( + self, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + property_chunk: Mapping[str, Any] = None, + ) -> Tuple[requests.PreparedRequest, requests.Response]: + request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + request = self._create_prepared_request( + path=self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + headers=dict(request_headers, **self.authenticator.get_auth_header()), + params=self.request_params( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, property_chunk=property_chunk + ), + json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + data=self.request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + ) + request_kwargs = self.request_kwargs(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + + response = self._send_request(request, request_kwargs) + return request, response class BulkSalesforceStream(SalesforceStream): @@ -406,10 +521,10 @@ def get_standard_instance(self) -> SalesforceStream: sobject_options=self.sobject_options, authenticator=self.authenticator, ) - new_cls: Type[SalesforceStream] = SalesforceStream + new_cls: Type[SalesforceStream] = RestSalesforceStream if isinstance(self, BulkIncrementalSalesforceStream): stream_kwargs.update({"replication_key": self.replication_key, "start_date": self.start_date}) - new_cls = IncrementalSalesforceStream + new_cls = IncrementalRestSalesforceStream return new_cls(**stream_kwargs) @@ -426,7 +541,7 @@ def transform_empty_string_to_none(instance: Any, schema: Any): return instance -class IncrementalSalesforceStream(SalesforceStream, ABC): +class IncrementalRestSalesforceStream(RestSalesforceStream, ABC): state_checkpoint_interval = 500 def __init__(self, replication_key: str, start_date: Optional[str], **kwargs): @@ -442,7 +557,11 @@ def format_start_date(start_date: Optional[str]) -> Optional[str]: return None def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + property_chunk: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: if next_page_token: """ @@ -450,12 +569,12 @@ def request_params( """ return {} - selected_properties = self.get_json_schema().get("properties", {}) + property_chunk = property_chunk or {} stream_date = stream_state.get(self.cursor_field) start_date = stream_date or self.start_date - query = f"SELECT {','.join(selected_properties.keys())} FROM {self.name} " + query = f"SELECT {','.join(property_chunk.keys())} FROM {self.name} " if start_date: query += f"WHERE {self.cursor_field} >= {start_date} " if self.name not in UNSUPPORTED_FILTERING_STREAMS: @@ -477,7 +596,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: latest_benchmark} -class BulkIncrementalSalesforceStream(BulkSalesforceStream, IncrementalSalesforceStream): +class BulkIncrementalSalesforceStream(BulkSalesforceStream, IncrementalRestSalesforceStream): def next_page_token(self, last_record: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: if self.name not in UNSUPPORTED_FILTERING_STREAMS: page_token: str = last_record[self.cursor_field] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 25a069b1cbfb..02a5e81db58d 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -19,8 +19,8 @@ CSV_FIELD_SIZE_LIMIT, BulkIncrementalSalesforceStream, BulkSalesforceStream, - IncrementalSalesforceStream, - SalesforceStream, + IncrementalRestSalesforceStream, + RestSalesforceStream, ) @@ -55,7 +55,7 @@ def test_bulk_stream_fallback_to_rest(mocker, requests_mock, stream_config, stre {"id": 11, "name": "custom entity", "created": "2020-01-02"} ] # mock REST API - mocker.patch("source_salesforce.source.SalesforceStream.read_records", Mock(return_value=rest_stream_records)) + mocker.patch("source_salesforce.source.RestSalesforceStream.read_records", Mock(return_value=rest_stream_records)) assert type(stream) is BulkIncrementalSalesforceStream assert list(stream.read_records(sync_mode=SyncMode.full_refresh)) == rest_stream_records @@ -196,12 +196,12 @@ def test_stream_start_date( def test_stream_start_date_should_be_converted_to_datetime_format(stream_config_date_format, stream_api): - stream: IncrementalSalesforceStream = generate_stream("ActiveFeatureLicenseMetric", stream_config_date_format, stream_api) + stream: IncrementalRestSalesforceStream = generate_stream("ActiveFeatureLicenseMetric", stream_config_date_format, stream_api) assert stream.start_date == "2010-01-18T00:00:00Z" def test_stream_start_datetime_format_should_not_changed(stream_config, stream_api): - stream: IncrementalSalesforceStream = generate_stream("ActiveFeatureLicenseMetric", stream_config, stream_api) + stream: IncrementalRestSalesforceStream = generate_stream("ActiveFeatureLicenseMetric", stream_config, stream_api) assert stream.start_date == "2010-01-18T21:18:20Z" @@ -335,8 +335,8 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): Next streams should not be executed. """ - stream_1: IncrementalSalesforceStream = generate_stream("KnowledgeArticle", stream_config, stream_api) - stream_2: IncrementalSalesforceStream = generate_stream("AcceptedEventRelation", stream_config, stream_api) + stream_1: IncrementalRestSalesforceStream = generate_stream("KnowledgeArticle", stream_config, stream_api) + stream_2: IncrementalRestSalesforceStream = generate_stream("AcceptedEventRelation", stream_config, stream_api) stream_1.state_checkpoint_interval = 3 configure_request_params_mock(stream_1, stream_2) @@ -397,7 +397,7 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): def test_pagination_rest(stream_config, stream_api): stream_name = "AcceptedEventRelation" - stream: SalesforceStream = generate_stream(stream_name, stream_config, stream_api) + stream: RestSalesforceStream = generate_stream(stream_name, stream_config, stream_api) stream.DEFAULT_WAIT_TIMEOUT_SECONDS = 6 # maximum wait timeout will be 6 seconds next_page_url = "/services/data/v52.0/query/012345" with requests_mock.Mocker() as m: @@ -558,7 +558,7 @@ def test_csv_field_size_limit(): def test_convert_to_standard_instance(stream_config, stream_api): bulk_stream = generate_stream("Account", stream_config, stream_api) rest_stream = bulk_stream.get_standard_instance() - assert isinstance(rest_stream, IncrementalSalesforceStream) + assert isinstance(rest_stream, IncrementalRestSalesforceStream) def test_bulk_stream_paging(stream_config, stream_api_pk): @@ -615,3 +615,67 @@ def get_query(request_index): q = f"{SELECT} WHERE (LastModifiedDate = {last_modified_date2} AND Id > '4') OR (LastModifiedDate > {last_modified_date2}) {ORDER_BY}" assert get_query(12) == q + + +def test_rest_stream_init_with_too_many_properties(stream_config, stream_api_v2_too_many_properties): + with pytest.raises(AssertionError): + # v2 means the stream is going to be a REST stream. + # A missing primary key is not allowed + generate_stream("Account", stream_config, stream_api_v2_too_many_properties) + + +def test_too_many_properties(stream_config, stream_api_v2_pk_too_many_properties, requests_mock): + stream = generate_stream("Account", stream_config, stream_api_v2_pk_too_many_properties) + chunks = len(list(stream.chunk_properties())) + assert stream.too_many_properties + assert stream.primary_key + assert type(stream) == RestSalesforceStream + url = next_page_url = "https://fase-account.salesforce.com/services/data/v52.0/queryAll" + requests_mock.get( + url, + [ + { + "json": { + "nextRecordsUrl": next_page_url, + "records": [{"Id": 1, "propertyA": "A"}, {"Id": 2, "propertyA": "A"}] + } + }, + { + "json": { + "nextRecordsUrl": next_page_url, + "records": [{"Id": 1, "propertyB": "B"}, {"Id": 2, "propertyB": "B"}] + } + }, + # 2 for 2 chunks above and 1 for a chunk below + *[{"json": {"records": [{"Id": 1}, {"Id": 2}], "nextRecordsUrl": next_page_url}} for _ in range(chunks - 3)], + { + "json": { + "records": [{"Id": 1}, {"Id": 2}] + } + }, + { + "json": { + "records": [{"Id": 3, "propertyA": "A"}, {"Id": 4, "propertyA": "A"}] + } + }, + { + "json": { + "records": [{"Id": 3, "propertyB": "B"}, {"Id": 4, "propertyB": "B"}] + } + }, + # 2 for 2 chunks above and 1 for a chunk below + *[{"json": {"records": [{"Id": 3}, {"Id": 4}]}} for _ in range(chunks - 3)], + { + "json": { + "records": [{"Id": 3}, {"Id": 4}] + } + } + ] + ) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) + assert records == [ + {"Id": 1, "propertyA": "A", "propertyB": "B"}, + {"Id": 2, "propertyA": "A", "propertyB": "B"}, + {"Id": 3, "propertyA": "A", "propertyB": "B"}, + {"Id": 4, "propertyA": "A", "propertyB": "B"} + ] diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py index f229b0b93c86..00a813d797ab 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py @@ -9,6 +9,7 @@ from airbyte_cdk.models import ConfiguredAirbyteCatalog from source_salesforce.api import Salesforce from source_salesforce.source import SourceSalesforce +from source_salesforce.streams import SalesforceStream @pytest.fixture(autouse=True) @@ -105,6 +106,26 @@ def stream_api_pk(stream_config): return _stream_api(stream_config, describe_response_data=describe_response_data) +@pytest.fixture(scope="module") +def stream_api_v2_too_many_properties(stream_config): + describe_response_data = { + "fields": [{"name": f"PropertyName{str(i)}", "type": "string"} for i in range(SalesforceStream.MAX_PROPERTIES_LENGTH)] + } + describe_response_data["fields"].extend([{"name": "BillingAddress", "type": "address"}]) + return _stream_api(stream_config, describe_response_data=describe_response_data) + + +@pytest.fixture(scope="module") +def stream_api_v2_pk_too_many_properties(stream_config): + describe_response_data = { + "fields": [{"name": f"PropertyName{str(i)}", "type": "string"} for i in range(SalesforceStream.MAX_PROPERTIES_LENGTH)] + } + describe_response_data["fields"].extend([ + {"name": "BillingAddress", "type": "address"}, {"name": "Id", "type": "string"} + ]) + return _stream_api(stream_config, describe_response_data=describe_response_data) + + def generate_stream(stream_name, stream_config, stream_api): return SourceSalesforce.generate_streams(stream_config, {stream_name: None}, stream_api)[0] diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index 9e29f60c2b90..c4288cfc048c 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -129,6 +129,7 @@ Now that you have set up the Salesforce source connector, check out the followin | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------| +| 2.0.1 | 2023-02-08 | [22597](https://github.com/airbytehq/airbyte/pull/22597) | Make multiple requests if a REST stream has too many properties | | 2.0.0 | 2023-02-02 | [22322](https://github.com/airbytehq/airbyte/pull/22322) | Remove `ActivityMetricRollup` stream | | 1.0.30 | 2023-01-27 | [22016](https://github.com/airbytehq/airbyte/pull/22016) | Set `AvailabilityStrategy` for streams explicitly to `None` | | 1.0.29 | 2023-01-05 | [20886](https://github.com/airbytehq/airbyte/pull/20886) | Remove `ActivityMetric` stream | From f330783e36a0552f4fd5ef47bc7b9f6647f8e466 Mon Sep 17 00:00:00 2001 From: darynaishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Thu, 9 Feb 2023 15:45:24 +0200 Subject: [PATCH 5/6] removed empty stream from testing (#22637) --- .../streams_with_output_records_catalog.json | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json index bb86bc48064e..e077ce8fb262 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json @@ -1,15 +1,5 @@ { "streams": [ - { - "stream": { - "name": "cohorts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "active_users", From 39051257b015763d538932dc47e06490c0d15b4e Mon Sep 17 00:00:00 2001 From: Joey Marshment-Howell Date: Thu, 9 Feb 2023 16:16:32 +0100 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Add=20TS=20CS?= =?UTF-8?q?S=20modules=20plugin=20(#22646)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add ts css modules plugin for autocomplete * Update vulnerable versions --------- Co-authored-by: Tim Roes --- airbyte-webapp/package.json | 6 + airbyte-webapp/pnpm-lock.yaml | 246 +++++++++++++++++++++++++++++++--- airbyte-webapp/tsconfig.json | 14 +- 3 files changed, 249 insertions(+), 17 deletions(-) diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 229d5e228fe8..0d1af97a546a 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -168,6 +168,7 @@ "tmpl": "^1.0.5", "ts-node": "^10.8.1", "typescript": "^4.9.5", + "typescript-plugin-css-modules": "^4.1.1", "vite": "^4.0.4", "vite-plugin-checker": "^0.5.5", "vite-plugin-svgr": "^2.4.0", @@ -202,6 +203,11 @@ "pnpm": { "patchedDependencies": { "react-virtualized@9.22.3": "patches/react-virtualized@9.22.3.patch" + }, + "overrides": { + "json5@<1.0.2": ">=1.0.2", + "postcss@<7.0.36": ">=7.0.36", + "loader-utils@<1.4.1": ">=1.4.1" } } } diff --git a/airbyte-webapp/pnpm-lock.yaml b/airbyte-webapp/pnpm-lock.yaml index 17f5e1af4a31..da9e981fa731 100644 --- a/airbyte-webapp/pnpm-lock.yaml +++ b/airbyte-webapp/pnpm-lock.yaml @@ -1,5 +1,10 @@ lockfileVersion: 5.4 +overrides: + json5@<1.0.2: '>=1.0.2' + postcss@<7.0.36: '>=7.0.36' + loader-utils@<1.4.1: '>=1.4.1' + patchedDependencies: react-virtualized@9.22.3: hash: wem7zdhrj6jola7ic3qcehiqii @@ -138,6 +143,7 @@ specifiers: ts-node: ^10.8.1 typesafe-actions: ^5.1.0 typescript: ^4.9.5 + typescript-plugin-css-modules: ^4.1.1 unified: ^10.1.2 unist-util-visit: ^4.1.0 url: ^0.11.0 @@ -288,6 +294,7 @@ devDependencies: tmpl: 1.0.5 ts-node: 10.9.1_cin3sed6ohfsopbmt6orxeb4o4 typescript: 4.9.5 + typescript-plugin-css-modules: 4.1.1_6qtx7vkbdhwvdm4crzlegk4mvi vite: 4.0.4_arwryhsn4zwmtf5pq2mmdxlt6a vite-plugin-checker: 0.5.5_svpeoejlom624h637oo4yy72dy vite-plugin-svgr: 2.4.0_vite@4.0.4 @@ -2305,7 +2312,7 @@ packages: resolution: {integrity: sha512-9hyNc4OmrXMtthDJq6zyJHll/UIYBWYmMG3rXty2eMeWxHWB0vlsq3AOI+k14PL15aSBAQolv0EZJWVJv/gCEg==} dependencies: '@firebase/util': 1.8.0 - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@firebase/database-compat/0.3.0_@firebase+app-types@0.9.0: @@ -2465,7 +2472,7 @@ packages: /@firebase/logger/0.4.0: resolution: {integrity: sha512-eRKSeykumZ5+cJPdxxJRgAC3G5NknY2GwEbKfymdnXtnT0Ucm4pspfR6GT4MUQEDuJwRVbVcSx85kgJulMoFFA==} dependencies: - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@firebase/messaging-compat/0.2.0_5z7svkifsmkn6ro3hru7lnxwrq: @@ -2648,7 +2655,7 @@ packages: /@formatjs/fast-memoize/1.2.7: resolution: {integrity: sha512-hPeM5LXUUjtCKPybWOUAWpv8lpja8Xz+uKprFPJcg5F2Rd+/bf1E0UUsLRpaAgOReAf5HMRtoIgv/UcyPICrTQ==} dependencies: - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@formatjs/icu-messageformat-parser/2.1.14: @@ -2663,7 +2670,7 @@ packages: resolution: {integrity: sha512-ND1ZkZfmLPcHjAH1sVpkpQxA+QYfOX3py3SjKWMUVGDow18gZ0WPqz3F+pJLYQMpS2LnnQ5zYR2jPVYTbRwMpg==} dependencies: '@formatjs/ecma402-abstract': 1.14.3 - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@formatjs/intl-displaynames/6.2.3: @@ -2685,7 +2692,7 @@ packages: /@formatjs/intl-localematcher/0.2.32: resolution: {integrity: sha512-k/MEBstff4sttohyEpXxCmC3MqbUn9VvHGlZ8fauLzkbwXmVrEeyzS+4uhrvAk9DWU9/7otYWxyDox4nT/KVLQ==} dependencies: - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@formatjs/intl/2.6.3_typescript@4.9.5: @@ -3185,7 +3192,7 @@ packages: '@motionone/easing': 10.15.1 '@motionone/types': 10.15.1 '@motionone/utils': 10.15.1 - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@motionone/dom/10.12.0: @@ -3203,7 +3210,7 @@ packages: resolution: {integrity: sha512-6hIHBSV+ZVehf9dcKZLT7p5PEKHGhDwky2k8RKkmOvUoYP3S+dXsKupyZpqx5apjd9f+php4vXk4LuS+ADsrWw==} dependencies: '@motionone/utils': 10.15.1 - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@motionone/generators/10.15.1: @@ -3211,7 +3218,7 @@ packages: dependencies: '@motionone/types': 10.15.1 '@motionone/utils': 10.15.1 - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@motionone/types/10.15.1: @@ -3223,7 +3230,7 @@ packages: dependencies: '@motionone/types': 10.15.1 hey-listen: 1.0.8 - tslib: 2.4.1 + tslib: 2.5.0 dev: false /@nicolo-ribaudo/eslint-scope-5-internals/5.1.1-v1: @@ -5551,7 +5558,7 @@ packages: esbuild: '>=0.10.0' dependencies: esbuild: 0.16.17 - tslib: 2.4.1 + tslib: 2.5.0 dev: true /abab/2.0.6: @@ -5920,7 +5927,7 @@ packages: resolution: {integrity: sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==} engines: {node: '>=4'} dependencies: - tslib: 2.4.1 + tslib: 2.5.0 dev: true /astral-regex/2.0.0: @@ -6725,6 +6732,12 @@ packages: engines: {node: '>= 0.6'} dev: true + /copy-anything/2.0.6: + resolution: {integrity: sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==} + dependencies: + is-what: 3.14.1 + dev: true + /copy-descriptor/0.1.1: resolution: {integrity: sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==} engines: {node: '>=0.10.0'} @@ -6790,6 +6803,13 @@ packages: hyphenate-style-name: 1.0.4 dev: false + /css-selector-tokenizer/0.7.3: + resolution: {integrity: sha512-jWQv3oCEL5kMErj4wRnK/OPoBi0D+P1FR2cDCKYPaMeD2eW3/mttav8HT4hT1CKopiJI/psEULjkClhvJo4Lvg==} + dependencies: + cssesc: 3.0.0 + fastparse: 1.1.2 + dev: true + /css-to-react-native/3.1.0: resolution: {integrity: sha512-AryfkFA29b4I3vG7N4kxFboq15DxwSXzhXM37XNEjwJMgjYIc8BcqfiprpAqX0zadI5PMByEIwAMzXxk5Vcc4g==} dependencies: @@ -7381,6 +7401,15 @@ packages: resolution: {integrity: sha512-Ds/TEoZjwggRoz/Q2O7SE3i4Jm66mqTDfmdHdq/7DKVk3bro9Q8h6WdXKdPqFLMoqxrDK5SVRzHVPOS6uuGtrg==} dev: true + /errno/0.1.8: + resolution: {integrity: sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==} + hasBin: true + requiresBuild: true + dependencies: + prr: 1.0.1 + dev: true + optional: true + /error-ex/1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} dependencies: @@ -8459,6 +8488,10 @@ packages: resolution: {integrity: sha512-bijHueCGd0LqqNK9b5oCMHc0MluJAx0cwqASgbWMvkO01lCYgIhacVRLcaDz3QnyYIRNJRDwMb41VuT6pHJ91Q==} dev: false + /fastparse/1.1.2: + resolution: {integrity: sha512-483XLLxTVIwWK3QTrMGRqUfUpoOs/0hbQrl2oz4J0pAcm3A3bu84wxTFqGqkJzewCLdME38xJLJAxBABfQT8sQ==} + dev: true + /fastq/1.15.0: resolution: {integrity: sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==} dependencies: @@ -8844,6 +8877,12 @@ packages: wide-align: 1.1.5 dev: true + /generic-names/1.0.3: + resolution: {integrity: sha512-b6OHfQuKasIKM9b6YPkX+KUj/TLBTx3B/1aT1T5F12FEuEqyFMdr59OMS53aoaSw8eVtapdqieX6lbg5opaOhA==} + dependencies: + loader-utils: 3.2.1 + dev: true + /gensync/1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -9345,6 +9384,21 @@ packages: safer-buffer: 2.1.2 dev: true + /icss-utils/3.0.1: + resolution: {integrity: sha512-ANhVLoEfe0KoC9+z4yiTaXOneB49K6JIXdS+yAgH0NERELpdIT7kkj2XxUPuHafeHnn8umXnECSpsfk1RTaUew==} + dependencies: + postcss: 8.4.21 + dev: true + + /icss-utils/5.1.0_postcss@8.4.21: + resolution: {integrity: sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + postcss: 8.4.21 + dev: true + /idb/7.0.1: resolution: {integrity: sha512-UUxlE7vGWK5RfB/fDwEGgRf84DY/ieqNha6msMV99UsEMQhJ1RwbCd8AYBj3QMgnE3VZnfQvm4oKVCJTYlqIgg==} dev: false @@ -9354,6 +9408,14 @@ packages: engines: {node: '>= 4'} dev: true + /image-size/0.5.5: + resolution: {integrity: sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==} + engines: {node: '>=0.10.0'} + hasBin: true + requiresBuild: true + dev: true + optional: true + /immer/9.0.19: resolution: {integrity: sha512-eY+Y0qcsB4TZKwgQzLaE/lqYMlKhv5J9dyd2RhhtGhNo2njPXDqU9XPfcNfa3MIDsdtZt5KlkIsirlo4dHsWdQ==} dev: true @@ -9797,6 +9859,10 @@ packages: get-intrinsic: 1.1.3 dev: true + /is-what/3.14.1: + resolution: {integrity: sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==} + dev: true + /is-windows/1.0.2: resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} engines: {node: '>=0.10.0'} @@ -10650,6 +10716,26 @@ packages: dotenv-expand: 10.0.0 dev: true + /less/4.1.3: + resolution: {integrity: sha512-w16Xk/Ta9Hhyei0Gpz9m7VS8F28nieJaL/VyShID7cYvP6IL5oHeL6p4TXSDJqZE/lNv0oJ2pGVjJsRkfwm5FA==} + engines: {node: '>=6'} + hasBin: true + dependencies: + copy-anything: 2.0.6 + parse-node-version: 1.0.1 + tslib: 2.5.0 + optionalDependencies: + errno: 0.1.8 + graceful-fs: 4.2.10 + image-size: 0.5.5 + make-dir: 2.1.0 + mime: 1.6.0 + needle: 3.2.0 + source-map: 0.6.1 + transitivePeerDependencies: + - supports-color + dev: true + /leven/3.1.0: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} engines: {node: '>=6'} @@ -10739,6 +10825,11 @@ packages: wrap-ansi: 7.0.0 dev: true + /loader-utils/3.2.1: + resolution: {integrity: sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw==} + engines: {node: '>= 12.13.0'} + dev: true + /locate-path/3.0.0: resolution: {integrity: sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==} engines: {node: '>=6'} @@ -10767,7 +10858,6 @@ packages: /lodash.camelcase/4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} - dev: false /lodash.debounce/4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} @@ -11667,6 +11757,20 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true + /needle/3.2.0: + resolution: {integrity: sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==} + engines: {node: '>= 4.4.x'} + hasBin: true + requiresBuild: true + dependencies: + debug: 3.2.7 + iconv-lite: 0.6.3 + sax: 1.2.4 + transitivePeerDependencies: + - supports-color + dev: true + optional: true + /negotiator/0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -12182,6 +12286,11 @@ packages: json-parse-even-better-errors: 2.3.1 lines-and-columns: 1.2.4 + /parse-node-version/1.0.1: + resolution: {integrity: sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==} + engines: {node: '>= 0.10'} + dev: true + /parse-srcset/1.0.2: resolution: {integrity: sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==} dev: false @@ -12310,6 +12419,48 @@ packages: engines: {node: '>=0.10.0'} dev: true + /postcss-filter-plugins/3.0.1: + resolution: {integrity: sha512-tRKbW4wWBEkSSFuJtamV2wkiV9rj6Yy7P3Y13+zaynlPEEZt8EgYKn3y/RBpMeIhNmHXFlSdzofml65hD5OafA==} + dependencies: + postcss: 8.4.21 + dev: true + + /postcss-icss-keyframes/0.2.1: + resolution: {integrity: sha512-4m+hLY5TVqoTM198KKnzdNudyu1OvtqwD+8kVZ9PNiEO4+IfHYoyVvEXsOHjV8nZ1k6xowf+nY4HlUfZhOFvvw==} + dependencies: + icss-utils: 3.0.1 + postcss: 8.4.21 + postcss-value-parser: 3.3.1 + dev: true + + /postcss-icss-selectors/2.0.3: + resolution: {integrity: sha512-dxFtq+wscbU9faJaH8kIi98vvCPDbt+qg1g9GoG0os1PY3UvgY1Y2G06iZrZb1iVC9cyFfafwSY1IS+IQpRQ4w==} + dependencies: + css-selector-tokenizer: 0.7.3 + generic-names: 1.0.3 + icss-utils: 3.0.1 + lodash: 4.17.21 + postcss: 8.4.21 + dev: true + + /postcss-load-config/3.1.4_aesdjsunmf4wiehhujt67my7tu: + resolution: {integrity: sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg==} + engines: {node: '>= 10'} + peerDependencies: + postcss: '>=8.0.9' + ts-node: '>=9.0.0' + peerDependenciesMeta: + postcss: + optional: true + ts-node: + optional: true + dependencies: + lilconfig: 2.0.5 + postcss: 8.4.21 + ts-node: 10.9.1_cin3sed6ohfsopbmt6orxeb4o4 + yaml: 1.10.2 + dev: true + /postcss-media-query-parser/0.2.3: resolution: {integrity: sha512-3sOlxmbKcSHMjlUXQZKQ06jOswE7oVkXPxmZdoB1r5l0q6gTFTQSHxNxOrCccElbW7dxNytifNEo8qidX2Vsig==} dev: true @@ -12344,7 +12495,6 @@ packages: /postcss-value-parser/3.3.1: resolution: {integrity: sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==} - dev: false /postcss-value-parser/4.2.0: resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} @@ -12521,6 +12671,11 @@ packages: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} dev: true + /prr/1.0.1: + resolution: {integrity: sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==} + dev: true + optional: true + /psl/1.9.0: resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} dev: true @@ -13221,7 +13376,7 @@ packages: ast-types: 0.14.2 esprima: 4.0.1 source-map: 0.6.1 - tslib: 2.4.1 + tslib: 2.5.0 dev: true /recast/0.23.1: @@ -13232,7 +13387,7 @@ packages: ast-types: 0.16.1 esprima: 4.0.1 source-map: 0.6.1 - tslib: 2.4.1 + tslib: 2.5.0 dev: true /recharts-scale/0.4.5: @@ -13471,6 +13626,10 @@ packages: resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} dev: true + /reserved-words/0.1.2: + resolution: {integrity: sha512-0S5SrIUJ9LfpbVl4Yzij6VipUdafHrOTzvmfazSw/jeZrZtQK303OPZW+obtkaw7jQlTQppy0UvZWm9872PbRw==} + dev: true + /reserved/0.1.2: resolution: {integrity: sha512-/qO54MWj5L8WCBP9/UNe2iefJc+L9yETbH32xO/ft/EYPOTCR5k+azvDUgdCOKwZH8hXwPd0b8XBL78Nn2U69g==} engines: {node: '>=0.8'} @@ -13658,6 +13817,10 @@ packages: immutable: 4.2.2 source-map-js: 1.0.2 + /sax/1.2.4: + resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} + dev: true + /saxes/6.0.0: resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} engines: {node: '>=v12.22.7'} @@ -14004,6 +14167,11 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + /source-map/0.7.4: + resolution: {integrity: sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==} + engines: {node: '>= 8'} + dev: true + /sourcemap-codec/1.4.8: resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==} deprecated: Please use @jridgewell/sourcemap-codec instead @@ -14475,6 +14643,19 @@ packages: resolution: {integrity: sha512-GP6WDNWf+o403jrEp9c5jibKavrtLW+/qYGhFxFrG8maXhwTBI7gLLhiBb0o7uFccWN+EOS9aMO6cGHWAO07OA==} dev: false + /stylus/0.59.0: + resolution: {integrity: sha512-lQ9w/XIOH5ZHVNuNbWW8D822r+/wBSO/d6XvtyHLF7LW4KaCIDeVbvn5DF8fGCJAUCwVhVi/h6J0NUcnylUEjg==} + hasBin: true + dependencies: + '@adobe/css-tools': 4.0.2 + debug: 4.3.4 + glob: 7.2.3 + sax: 1.2.4 + source-map: 0.7.4 + transitivePeerDependencies: + - supports-color + dev: true + /supports-color/5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -14785,6 +14966,15 @@ packages: strip-bom: 3.0.0 dev: true + /tsconfig-paths/4.1.2: + resolution: {integrity: sha512-uhxiMgnXQp1IR622dUXI+9Ehnws7i/y6xvpZB9IbUVOPy0muvdvgXeZOn88UcGPiT98Vp3rJPTa8bFoalZ3Qhw==} + engines: {node: '>=6'} + dependencies: + json5: 2.2.3 + minimist: 1.2.7 + strip-bom: 3.0.0 + dev: true + /tslib/1.14.1: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==} @@ -14793,7 +14983,6 @@ packages: /tslib/2.5.0: resolution: {integrity: sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==} - dev: true /tsutils/3.21.0_typescript@4.9.5: resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} @@ -14884,6 +15073,31 @@ packages: engines: {node: '>= 4'} dev: false + /typescript-plugin-css-modules/4.1.1_6qtx7vkbdhwvdm4crzlegk4mvi: + resolution: {integrity: sha512-kpVxGkY/go9eV5TP1YUDJ6SqwBx2OIuVStMCxKyg9PhJVFXjLYR7AuItVLwoz0NCdiemH91WhtgAjb96jI34DA==} + peerDependencies: + typescript: '>=3.9.0' + dependencies: + dotenv: 16.0.3 + icss-utils: 5.1.0_postcss@8.4.21 + less: 4.1.3 + lodash.camelcase: 4.3.0 + postcss: 8.4.21 + postcss-filter-plugins: 3.0.1 + postcss-icss-keyframes: 0.2.1 + postcss-icss-selectors: 2.0.3 + postcss-load-config: 3.1.4_aesdjsunmf4wiehhujt67my7tu + reserved-words: 0.1.2 + sass: 1.57.1 + source-map-js: 1.0.2 + stylus: 0.59.0 + tsconfig-paths: 4.1.2 + typescript: 4.9.5 + transitivePeerDependencies: + - supports-color + - ts-node + dev: true + /typescript/4.9.5: resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} engines: {node: '>=4.2.0'} diff --git a/airbyte-webapp/tsconfig.json b/airbyte-webapp/tsconfig.json index ec543a1239f2..902cf2c91a51 100644 --- a/airbyte-webapp/tsconfig.json +++ b/airbyte-webapp/tsconfig.json @@ -26,7 +26,19 @@ "noImplicitThis": true, "noImplicitAny": true, "noFallthroughCasesInSwitch": true, - "useUnknownInCatchVariables": false + "useUnknownInCatchVariables": false, + "plugins": [ + { + "name": "typescript-plugin-css-modules", + "options": { + "rendererOptions": { + "sass": { + "loadPaths": ["./src"] + } + } + } + } + ] }, "include": ["src"] }