bugfix/redshift-large-int (#117)

* bugfix/redshift-large-int * regen docs * final adjustments before review * Update CHANGELOG.md Co-authored-by: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> * Update CHANGELOG.md Co-authored-by: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> * Update CHANGELOG.md Co-authored-by: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> --------- Co-authored-by: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Co-authored-by: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
fivetran · Mar 11, 2024 · 2fe07aa · 2fe07aa
1 parent 1627958
commit 2fe07aa
Show file tree

Hide file tree

Showing 11 changed files with 56 additions and 37 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,22 @@
+# dbt_fivetran_log v1.6.0
+[PR #117](https://github.com/fivetran/dbt_fivetran_log/pull/117) includes the following updates as a result of users encountering numeric counts exceeding the limit of a standard integer. Therefore, these fields were required to be cast as `bigint` in order to avoid "integer out of range" errors:
+
+## Breaking Changes
+> ⚠️ Since the following changes result in a field changing datatype, we recommend running a `--full-refresh` after upgrading to this version to avoid possible incremental failures.
+- The following fields in the `fivetran_platform__audit_table` model have been updated to be cast as `dbt.type_bigint()` (previously was `dbt.type_int()`)
+  - `sum_rows_replaced_or_inserted`
+  - `sum_rows_updated`
+  - `sum_rows_deleted`
+
+## Bug Fixes
+- The following fields in the `fivetran_platform__connector_daily_events` model have been updated to be cast as `dbt.type_bigint()` (previously was `dbt.type_int()`)
+  - `count_api_calls`
+  - `count_record_modifications`
+  - `count_schema_changes`
+
+## Under the Hood
+- Modified `log` seed data within the integration tests folder to ensure that large integers are being tested as part of our integration tests.
+
 # dbt_fivetran_log v1.5.0
 [PR #114](https://github.com/fivetran/dbt_fivetran_log/pull/114) includes the following updates:
 

diff --git a/README.md b/README.md
@@ -66,7 +66,7 @@ Include the following Fivetran Platform package version range in your `packages.
 ```yaml
 packages:
   - package: fivetran/fivetran_log
-    version: [">=1.5.0", "<1.6.0"]
+    version: [">=1.6.0", "<1.7.0"]
 ```
 
 > Note that altough the source connector is now "Fivetran Platform", the package retains the old name of "fivetran_log".

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '1.5.0'
+version: '1.6.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '1.5.0'
+version: '1.6.0'
 
 config-version: 2
 profile: 'integration_tests'

diff --git a/integration_tests/seeds/log.csv b/integration_tests/seeds/log.csv
@@ -1,32 +1,32 @@
 id,time_stamp,_fivetran_synced,connector_id,event,message_data,message_event,transformation_id,sync_id
 9Gktalz7uFZbVYKR7zlDeUd4nSM=,2021-02-13 13:34:03.343,2021-02-16 21:11:07.065,legislative_lazy,WARNING,"{""type"":""table_excluded_by_system"",""message"":""salesforce.affectlayer__AffectLayer_User_Setting__ChangeEvent has been Excluded by system. Reason : Not queryable""}",warning,123,456abc
 f7EVPLOIMhuOs45jlXWN3lutVEc=,2021-02-12 08:15:05.555,2021-02-16 21:48:51.493,paint_wedges,INFO,"{""method"":""GET"",""uri"":""https://google.com""}",api_call,123,456abc
-fJk+tL6p9gQE83WaCPdevLs9FXU=,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,intrinsic_departed,INFO,,sync_end,123,456abc
-f7UkkDoGwEVnG7M2tFQFy7FSoDM=,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified,123,456abc
-qAihN0e5BULEPL7M9uwJ6Kyy9x8=,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,123,456abc
-OblnLHyCtL16oC4niA5+tYnOPo0=,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,123,456abc
-dfjD9Lt4zcYcKSd/XOxCXjDnKhg=,2021-12-09 14:26:44,2021-12-09 20:30:53.952,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_start,123,456abc
-tPQov43eYhIesRb7bHvmhlk6kr4=,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified,123,456abc
-03F1Ja3wG4EAjlPUHyYS6UPSE3I=,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,123,456abc
-6dq4Y552vJGY0ONeD4WZu6Lodvo=,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,123,456abc
-BT3RzERiPYzSQs99qeL8bNO9ScE=,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,123,456abc
-B+GRNp7ayvSHYnWGoq9b7vNaKW4=,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,123,456abc
-TCvFRB2TzpEiVV3y8tUHdhuqdhU=,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,123,456abc
-Ps84wKmF1+daFzbwsOU7j7Zb4MY=,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified,123,456abc
-KDeFiMYLlB+GIFzQU9Huc3Qj6eo=,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":3,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified,123,456abc
-nZdGbakpx39P1l2HOYUzxh9l5Vw=,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,123,456abc
-KY3fOFG038Dsx7OTowE5jYw7kBE=,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,123,456abc
-wVCLYJ0JurBy4JaN5odnOOY94oQ=,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,123,456abc
-jkLxnPUfQ/9mHaWCmxUppphul8ZA=,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,123,456abc
-TAnUngGvt/9+vOSde9OI357BEnE=,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_start,123,456abc
-H6AsShJjRoy1n75oVRxjQ08perA=,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_start,123,456abc
-bCgsj8twLxBBhaiBN9N4S35grkI=,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_start,123,456abc
-UcIZasAY/eulDQF2SQM6Y5W83gU=,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start,123,456abc
-mm0QUb+ldnbSWknrBrF/v24YGvo=,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,intrinsic_departed,INFO,,sync_start,123,456abc
-Z/kZAi7u6DzO7UrZeWK/ZsAkbno=,2021-12-10 14:27:00.504,2021-12-10 20:30:53.959,intrinsic_departed,INFO,,sync_end,,456abc
-+IX63qGrDFPaTy9/6epjpppk4JI=,2021-12-10 14:26:41.875,2021-12-10 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,123,456abc
-ZZLSXVYxirh8aKlgdCzo5fvd71g=,2021-12-10 14:26:42.029,2021-12-10 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,123,456abc
-D7UqnKYn6OT04HkUcPNjXA95ttI=,2021-12-10 14:26:29.719,2021-12-10 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start,123,456abc
-V3H62Gj5c6nB8bY+OdxMHvp/hNs=,2021-12-10 14:26:05.907,2021-12-10 20:30:53.778,intrinsic_departed,INFO,,sync_start,123,456abc
+fJk+tL6p9gQE83WaCPdevLs9FXU=,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,this_connector,INFO,,sync_end,123,456abc
+f7UkkDoGwEVnG7M2tFQFy7FSoDM=,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,this_connector,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified,123,456abc
+qAihN0e5BULEPL7M9uwJ6Kyy9x8=,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,this_connector,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,123,456abc
+OblnLHyCtL16oC4niA5+tYnOPo0=,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,this_connector,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,123,456abc
+dfjD9Lt4zcYcKSd/XOxCXjDnKhg=,2021-12-09 14:26:44,2021-12-09 20:30:53.952,this_connector,INFO,"{""table"":""fivetran_audit""}",write_to_table_start,123,456abc
+tPQov43eYhIesRb7bHvmhlk6kr4=,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,this_connector,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified,123,456abc
+03F1Ja3wG4EAjlPUHyYS6UPSE3I=,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,this_connector,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,123,456abc
+6dq4Y552vJGY0ONeD4WZu6Lodvo=,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,this_connector,INFO,"{""table"":""media_history""}",write_to_table_end,123,456abc
+BT3RzERiPYzSQs99qeL8bNO9ScE=,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,this_connector,INFO,"{""table"":""media_insights""}",write_to_table_end,123,456abc
+B+GRNp7ayvSHYnWGoq9b7vNaKW4=,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,this_connector,INFO,"{""table"":""media_history""}",write_to_table_end,123,456abc
+TCvFRB2TzpEiVV3y8tUHdhuqdhU=,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,this_connector,INFO,"{""table"":""media_insights""}",write_to_table_end,123,456abc
+Ps84wKmF1+daFzbwsOU7j7Zb4MY=,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,this_connector,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified,123,456abc
+KDeFiMYLlB+GIFzQU9Huc3Qj6eo=,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,this_connector,INFO,"{""schema"":""instagram_business"",""count"":11624686068,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified,123,456abc
+nZdGbakpx39P1l2HOYUzxh9l5Vw=,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,this_connector,INFO,"{""table"":""user_insights""}",write_to_table_end,123,456abc
+KY3fOFG038Dsx7OTowE5jYw7kBE=,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,this_connector,INFO,"{""table"":""user_history""}",write_to_table_end,123,456abc
+wVCLYJ0JurBy4JaN5odnOOY94oQ=,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,this_connector,INFO,"{""table"":""user_history""}",write_to_table_end,123,456abc
+jkLxnPUfQ/9mHaWCmxUppphul8ZA=,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,this_connector,INFO,"{""table"":""user_insights""}",write_to_table_end,123,456abc
+TAnUngGvt/9+vOSde9OI357BEnE=,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,this_connector,INFO,"{""table"":""media_history""}",write_to_table_start,123,456abc
+H6AsShJjRoy1n75oVRxjQ08perA=,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,this_connector,INFO,"{""table"":""user_insights""}",write_to_table_start,123,456abc
+bCgsj8twLxBBhaiBN9N4S35grkI=,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,this_connector,INFO,"{""table"":""user_history""}",write_to_table_start,123,456abc
+UcIZasAY/eulDQF2SQM6Y5W83gU=,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,this_connector,INFO,"{""table"":""media_insights""}",write_to_table_start,123,456abc
+mm0QUb+ldnbSWknrBrF/v24YGvo=,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,this_connector,INFO,,sync_start,123,456abc
+Z/kZAi7u6DzO7UrZeWK/ZsAkbno=,2021-12-10 14:27:00.504,2021-12-10 20:30:53.959,this_connector,INFO,,sync_end,,456abc
++IX63qGrDFPaTy9/6epjpppk4JI=,2021-12-10 14:26:41.875,2021-12-10 20:30:53.948,this_connector,INFO,"{""table"":""media_insights""}",write_to_table_end,123,456abc
+ZZLSXVYxirh8aKlgdCzo5fvd71g=,2021-12-10 14:26:42.029,2021-12-10 20:30:53.949,this_connector,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,123,456abc
+D7UqnKYn6OT04HkUcPNjXA95ttI=,2021-12-10 14:26:29.719,2021-12-10 20:30:53.878,this_connector,INFO,"{""table"":""media_insights""}",write_to_table_start,123,456abc
+V3H62Gj5c6nB8bY+OdxMHvp/hNs=,2021-12-10 14:26:05.907,2021-12-10 20:30:53.778,this_connector,INFO,,sync_start,123,456abc
 jkLxnPUfQ/9mHaWCmxUQEhul8ZA=,2023-11-09 11:31:31.579000,2023-11-09 15:55:12.560000,protestations_mourned,INFO,"{""actor"":""me@me.com"",""properties"":{""endpoint"":""************"",""apiKey"":""************"",""customEvents"":[],""syncMode"":""AllEvents"",""isE2ETest"":false,""events"":[],""customEventSyncMode"":""AllEvents""},""id"":""iterable""}",create_connector,,
 R7UqnKYn6OT04HkUcPNjXA95qqI=,2021-12-10 16:26:29.719,2021-12-10 20:30:53.878,intrinsic_departed,INFO,says actor but not a json,status,,456abc
diff --git a/models/fivetran_platform__audit_table.sql b/models/fivetran_platform__audit_table.sql
@@ -83,7 +83,7 @@ records_modified_log as (
         {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['table']) }} as table_name,
         {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['schema']) }} as schema_name,
         {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['operationType']) }} as operation_type,
-        cast ({{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['count']) }} as {{ dbt.type_int() }}) as row_count
+        cast ({{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['count']) }} as {{ dbt.type_bigint() }}) as row_count
     from sync_log 
     where event_subtype = 'records_modified'
 

diff --git a/models/fivetran_platform__connector_daily_events.sql b/models/fivetran_platform__connector_daily_events.sql
@@ -16,7 +16,7 @@ log_events as (
             when event_subtype in ('create_table', 'alter_table', 'create_schema', 'change_schema_config') then 'schema_change' 
             else event_subtype end as event_subtype,
 
-        sum(case when event_subtype = 'records_modified' then cast( {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['count']) }} as {{ dbt.type_int()}} )
+        sum(case when event_subtype = 'records_modified' then cast( {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['count']) }} as {{ dbt.type_bigint()}} )
         else 1 end) as count_events 
 
     from {{ ref('stg_fivetran_platform__log') }}