From 4fd3ba8cb52e62eff85fc98f1e26a357c219faa9 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 14 Dec 2022 12:09:42 -0800 Subject: [PATCH 01/12] [faker] decouple stream state --- .../connectors/source-faker/Dockerfile | 2 +- .../source-faker/source_faker/streams.py | 42 ++++++++++--------- docs/integrations/sources/faker.md | 1 + 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile index 17f51a82818d..3292aba263de 100644 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -34,5 +34,5 @@ COPY source_faker ./source_faker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 352dfb2411db..f2fc1bc81816 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -142,15 +142,15 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: break self.state = {self.cursor_field: total_records, "seed": self.seed} - set_total_user_records(total_records) class Purchases(Stream, IncrementalMixin): primary_key = None - cursor_field = "user_id" + cursor_field = "id" - def __init__(self, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count:int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) + self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice @@ -222,23 +222,33 @@ def generate_purchases(self, user_id: int, purchases_count: int) -> list[Dict]: return purchases def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: - purchases_count = self.state[self.cursor_field] if self.cursor_field in self.state else 0 - - if total_user_records <= 0: - return # if there are no new users, there should be no new purchases + total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 + total_user_records = self.state['user_id'] if 'user_id' in self.state else 0 + user_records_in_sync = 0 + user_records_in_slice = 0 median_record_byte_size = 230 yield generate_estimate( - self.name, total_user_records - purchases_count * 1.3, median_record_byte_size + self.name, (self.count - total_user_records) * 1.3, median_record_byte_size ) # a fuzzy guess, some users have purchases, some don't - for i in range(purchases_count, total_user_records): - purchases = self.generate_purchases(i + 1, purchases_count) + for i in range(total_user_records, self.count): + purchases = self.generate_purchases(i + 1, total_purchase_records) for purchase in purchases: + total_purchase_records += 1 yield purchase - purchases_count += 1 + total_user_records += 1 + user_records_in_sync += 1 + user_records_in_slice += 1 + + if user_records_in_slice >= self.records_per_slice: + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + user_records_in_slice = 0 + + if user_records_in_sync == self.records_per_sync: + break - self.state = {self.cursor_field: total_user_records, "seed": self.seed} + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} def generate_estimate(stream_name: str, total: int, bytes_per_row: int): @@ -247,11 +257,3 @@ def generate_estimate(stream_name: str, total: int, bytes_per_row: int): type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) ) return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) - - -# a globals hack to share data between streams: -total_user_records = 0 - - -def set_total_user_records(total: int): - globals()["total_user_records"] = total diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index 4ea89e33da23..c278d92ea0ab 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -83,6 +83,7 @@ None! | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 1.0.0 | 2022-12-14 | [xxx](https://github.com/airbytehq/airbyte/pull/xxx) | Decouple stream states for better parallelism | | 1.0.0 | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490) | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename | | 0.2.1 | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197) | Emit `AirbyteEstimateTraceMessage` | | 0.2.0 | 2022-10-14 | [18021](https://github.com/airbytehq/airbyte/pull/18021) | Move to mimesis for speed! | From 5ab75ff0b6ab6759932fc73e80d1c84fb0f71f69 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 14 Dec 2022 12:10:46 -0800 Subject: [PATCH 02/12] add PR # --- docs/integrations/sources/faker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index c278d92ea0ab..49e1a6192aed 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -83,7 +83,7 @@ None! | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | -| 1.0.0 | 2022-12-14 | [xxx](https://github.com/airbytehq/airbyte/pull/xxx) | Decouple stream states for better parallelism | +| 1.0.0 | 2022-12-14 | [20492](https://github.com/airbytehq/airbyte/pull/20492) | Decouple stream states for better parallelism | | 1.0.0 | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490) | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename | | 0.2.1 | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197) | Emit `AirbyteEstimateTraceMessage` | | 0.2.0 | 2022-10-14 | [18021](https://github.com/airbytehq/airbyte/pull/18021) | Move to mimesis for speed! | From 5333c1beb5da59f4cad1eaef471f7f7ed7d8724c Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 14 Dec 2022 13:20:23 -0800 Subject: [PATCH 03/12] commit Stream instantiate changes --- .../connectors/source-faker/source_faker/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index ef56bbfc1925..25a35776e34e 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -28,5 +28,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ Products(count, seed, records_per_sync, records_per_slice), Users(count, seed, records_per_sync, records_per_slice), - Purchases(seed, records_per_sync, records_per_slice), + Purchases(count, seed, records_per_sync, records_per_slice), ] From 5466f3165d8b744980287633f97b08bfe2c71f35 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 14 Dec 2022 17:12:44 -0800 Subject: [PATCH 04/12] fixup expected record --- .../source-faker/integration_tests/expected_records.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index 5ade6ae247df..87ff4c896424 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -18,7 +18,7 @@ {"stream": "purchases", "data": {"id": 8, "product_id": 67, "user_id": 6, "added_to_cart_at": "2008-03-02T18:14:15+00:00", "purchased_at": "2020-06-21T18:14:15+00:00", "returned_at": "2020-09-24T18:14:15+00:00"}, "emitted_at": 1669830193010} {"stream": "purchases", "data": {"id": 9, "product_id": 91, "user_id": 7, "added_to_cart_at": "2022-03-12T17:13:51+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} {"stream": "purchases", "data": {"id": 10, "product_id": 79, "user_id": 8, "added_to_cart_at": "2017-12-31T07:18:11+00:00", "purchased_at": "2019-05-14T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 11, "product_id": 91, "user_id": 8, "added_to_cart_at": "2022-03-24T07:18:11+00:00", "purchased_at": "2022-05-11T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} +{"stream": "purchases", "data": {"id": 11, "product_id": 91, "user_id": 8, "added_to_cart_at": "2022-03-24T07:18:11+00:00", "purchased_at": "2022-06-29T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} {"stream": "purchases", "data": {"id": 12, "product_id": 19, "user_id": 9, "added_to_cart_at": "2020-11-29T04:56:09+00:00", "purchased_at": "2022-03-02T04:56:09+00:00", "returned_at": "2022-04-12T04:56:09+00:00"}, "emitted_at": 1669830193010} {"stream": "purchases", "data": {"id": 13, "product_id": 63, "user_id": 10, "added_to_cart_at": "2003-08-05T17:43:25+00:00", "purchased_at": "2015-12-15T17:43:25+00:00", "returned_at": null}, "emitted_at": 1669830193010} {"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1669830193011} From 5ca497f830bb219e183baca90d4b1905320fdb36 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 15 Dec 2022 11:45:46 -0800 Subject: [PATCH 05/12] skip backward test for this version too --- .../connectors/source-faker/acceptance-test-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml index f6aed27fd225..023df794897b 100644 --- a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml @@ -14,7 +14,7 @@ acceptance_tests: tests: - config_path: secrets/config.json backward_compatibility_tests_config: - disable_for_version: "0.2.1" + disable_for_version: "1.0.0" basic_read: tests: - config_path: secrets/config.json From fb408a8b5dd76f69ef011621d2ca7f794ce9e85f Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Fri, 16 Dec 2022 14:18:30 -0800 Subject: [PATCH 06/12] Apply suggestions from code review Co-authored-by: Augustin --- airbyte-integrations/connectors/source-faker/Dockerfile | 2 +- .../connectors/source-faker/acceptance-test-config.yml | 2 +- .../connectors/source-faker/source_faker/streams.py | 2 +- docs/integrations/sources/faker.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile index 3292aba263de..1e2a25e003d3 100644 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -34,5 +34,5 @@ COPY source_faker ./source_faker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml index 023df794897b..f9eaae62331b 100644 --- a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml @@ -14,7 +14,7 @@ acceptance_tests: tests: - config_path: secrets/config.json backward_compatibility_tests_config: - disable_for_version: "1.0.0" + disable_for_version: "1.0.0" # We changed the cursor field of the Purchases stream in 2.0.0 basic_read: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index f2fc1bc81816..f909177ad920 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -223,7 +223,7 @@ def generate_purchases(self, user_id: int, purchases_count: int) -> list[Dict]: def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 - total_user_records = self.state['user_id'] if 'user_id' in self.state else 0 + total_user_records = self.state["user_id"] if "user_id" in self.state else 0 user_records_in_sync = 0 user_records_in_slice = 0 diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index 49e1a6192aed..337434ce4b1c 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -83,7 +83,7 @@ None! | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | -| 1.0.0 | 2022-12-14 | [20492](https://github.com/airbytehq/airbyte/pull/20492) | Decouple stream states for better parallelism | +| 2.0.0 | 2022-12-14 | [20492](https://github.com/airbytehq/airbyte/pull/20492) | Decouple stream states for better parallelism | | 1.0.0 | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490) | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename | | 0.2.1 | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197) | Emit `AirbyteEstimateTraceMessage` | | 0.2.0 | 2022-10-14 | [18021](https://github.com/airbytehq/airbyte/pull/18021) | Move to mimesis for speed! | From 2a967ae5db1b528bfb1123f7ae81a0c916617ca0 Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 16 Dec 2022 17:34:04 -0800 Subject: [PATCH 07/12] lint --- airbyte-integrations/connectors/source-faker/main.py | 1 - .../connectors/source-faker/source_faker/streams.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/main.py b/airbyte-integrations/connectors/source-faker/main.py index eb00656da52e..15947dc56432 100644 --- a/airbyte-integrations/connectors/source-faker/main.py +++ b/airbyte-integrations/connectors/source-faker/main.py @@ -1,7 +1,6 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -# import sys diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index f909177ad920..eab4969305f5 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -148,7 +148,7 @@ class Purchases(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count:int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed From f178b1465fddc8450172e2aa5a6fb7ad6cfa7c9c Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Thu, 22 Dec 2022 14:54:22 -0800 Subject: [PATCH 08/12] Create realistic datasets of 10GB, 100GB, and 1TB in size (#20558) * Faker CSV Streaming utilities * readme * don't do a final pipe to jq or you will run out or ram * doc --- .../source-faker/csv_export/README.md | 76 +++++++++++++++++++ .../source-faker/csv_export/main.sh | 9 +++ .../source-faker/csv_export/products.sh | 13 ++++ .../source-faker/csv_export/purchases.sh | 13 ++++ .../source-faker/csv_export/users.sh | 13 ++++ 5 files changed, 124 insertions(+) create mode 100644 airbyte-integrations/connectors/source-faker/csv_export/README.md create mode 100755 airbyte-integrations/connectors/source-faker/csv_export/main.sh create mode 100755 airbyte-integrations/connectors/source-faker/csv_export/products.sh create mode 100755 airbyte-integrations/connectors/source-faker/csv_export/purchases.sh create mode 100755 airbyte-integrations/connectors/source-faker/csv_export/users.sh diff --git a/airbyte-integrations/connectors/source-faker/csv_export/README.md b/airbyte-integrations/connectors/source-faker/csv_export/README.md new file mode 100644 index 000000000000..fdc5245ff308 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/README.md @@ -0,0 +1,76 @@ +# Python Source CSV Export + +This collection of tools is used to run the source and capture it's AirbyteMessages and convert them into CSV files. This is useful if you want to manually inspect this data or load it into a database manually. + +To be fast, we make use of parallel processing per-stream and only using command-line tools. This works by the main file (`main.sh`) running the source via python and tee-ing the output of RECORDS to sub-scripts which use `jq` to convert the records into CSV-delimited output, which we finally write to disk. + +As we read the connector config files, e.g. `--config secrets/config.json --state secrets/state.json --catalog integration_tests/configured_catalog.json`, you can manually step forward your sync if you need to read and store the input in chunks. + +## The road to 1TB of faker data + +There's commentary on this at https://github.com/airbytehq/airbyte/pull/20558, along with some cool SQL tricks. + +- 2 Billion faker users for 1TB: `10,000,000*(1024/5.02) = 2,039,840,637` +- 200 Million faker users for 100GB: `10,000,000*(100/5.02) = 199,203,187` +- 20 Million faker users for 10GB: `10,000,000*(10/5.02) = 19,920,318` + +But let's assume we don't have 1TB of local hard disk. So, we want to make 10 chunks of data, each around 100GB in size. + +**`config.json`** + +```json +{ + "count": 2039840637, + "seed": 0, + "records_per_sync": 203984064 +} +``` + +**`state.json`** + +At the end of every sync, increment the `id` in the users stream and the `user_id` in the purchases stream by `203984064`, the `records_per_sync` chunk size + +```json +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 0 + }, + "stream_descriptor": { + "name": "users" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 0, + "user_id": 0 + }, + "stream_descriptor": { + "name": "purchases" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 0 + }, + "stream_descriptor": { + "name": "products" + } + } + } +] +``` + +Finally, ensure that you've opted-into all the streams in `integration_tests/configured_catalog.json` + +## TODO + +- This is currently set up very manually, in that we build bash scripts for each stream and manually populate the header information. This information all already lives in the connector's catalog. We probably could build these bash files on-demand with a python script... diff --git a/airbyte-integrations/connectors/source-faker/csv_export/main.sh b/airbyte-integrations/connectors/source-faker/csv_export/main.sh new file mode 100755 index 000000000000..c9cd903f302d --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/main.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" +cd ".." + +mkdir -p /tmp/csv + +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json --state secrets/state.json \ + | tee >(./csv_export/purchases.sh) >(./csv_export/products.sh) >(./csv_export/users.sh) diff --git a/airbyte-integrations/connectors/source-faker/csv_export/products.sh b/airbyte-integrations/connectors/source-faker/csv_export/products.sh new file mode 100755 index 000000000000..505340e73919 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/products.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" + +FILE="/tmp/csv/products.csv" + +rm -rf $FILE + +echo "make, model, price, created_at" >> $FILE + +jq -c 'select((.type | contains("RECORD")) and (.record.stream | contains("products"))) .record.data' \ + | jq -r '[.make, .model, .price, .created_at] | @csv' \ + >> $FILE diff --git a/airbyte-integrations/connectors/source-faker/csv_export/purchases.sh b/airbyte-integrations/connectors/source-faker/csv_export/purchases.sh new file mode 100755 index 000000000000..9a79e225fdeb --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/purchases.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" + +FILE="/tmp/csv/purchases.csv" + +rm -rf $FILE + +echo "id, product_id, user_id, added_to_cart_at, purchased_at, returned_at" >> $FILE + +jq -c 'select((.type | contains("RECORD")) and (.record.stream | contains("purchases"))) .record.data' \ + | jq -r '[.id, .product_id, .user_id, .added_to_cart_at, .purchased_at, .returned_at] | @csv' \ + >> $FILE diff --git a/airbyte-integrations/connectors/source-faker/csv_export/users.sh b/airbyte-integrations/connectors/source-faker/csv_export/users.sh new file mode 100755 index 000000000000..a6ac8f374b1d --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/csv_export/users.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" + +FILE="/tmp/csv/users.csv" + +rm -rf $FILE + +echo "id, created_at, updated_at, name, title, age, email, telephone, gender, language, academic_degree, nationality, occupation, height, blood_type, weight" >> $FILE + +jq -c 'select((.type | contains("RECORD")) and (.record.stream | contains("users"))) .record.data' \ + | jq -r '[.id, .created_at, .updated_at, .name, .title, .age, .email, .telephone, .gender, .language, .academic_degree, .nationality, .occupation, .height, .blood_type, .weight] | @csv' \ + >> $FILE From fe328c3880cdce612043c64293923cc1e4894971 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Mon, 2 Jan 2023 14:33:51 -0800 Subject: [PATCH 09/12] Faker gets 250% faster (#20741) * Faker is 250% faster * threads in spec + lint * pass tests * revert changes to record helper * cleanup * update expected_records * bump default records-per-slice to 1k * enforce unique email addresses * cleanup * more comments * `parallelism` and pass tests * update expected records * cleanup notes --- .../integration_tests/expected_records.jsonl | 243 +++++++++--------- .../airbyte_message_with_cached_json.py | 22 ++ .../source_faker/purchase_generator.py | 103 ++++++++ .../source-faker/source_faker/source.py | 8 +- .../source-faker/source_faker/spec.json | 10 +- .../source-faker/source_faker/streams.py | 183 ++++--------- .../source_faker/user_generator.py | 72 ++++++ .../source-faker/source_faker/utils.py | 14 + .../source-faker/unit_tests/unit_test.py | 22 +- 9 files changed, 408 insertions(+), 269 deletions(-) create mode 100644 airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py create mode 100644 airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py create mode 100644 airbyte-integrations/connectors/source-faker/source_faker/user_generator.py diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index 87ff4c896424..e6e1c3e60c9a 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -1,123 +1,120 @@ -{"stream": "users", "data": {"id": 1, "created_at": "2009-08-12T18:57:58+00:00", "updated_at": "2012-07-02T08:32:31+00:00", "name": "Reda", "title": "M.Sc.Tech.", "age": 47, "email": "locations1983@protonmail.com", "telephone": "+1-(110)-795-7610", "gender": "Male", "language": "Tamil", "academic_degree": "Master", "nationality": "Italian", "occupation": "Word Processing Operator", "height": "1.55", "blood_type": "B\u2212", "weight": 58}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 2, "created_at": "2008-09-23T19:57:09+00:00", "updated_at": "2016-03-10T04:48:06+00:00", "name": "Tristan", "title": "M.Sc.Tech.", "age": 32, "email": "variations1847@duck.com", "telephone": "683-770-9281", "gender": "Other", "language": "Bosnian", "academic_degree": "Bachelor", "nationality": "Estonian", "occupation": "Tiler", "height": "2.00", "blood_type": "AB\u2212", "weight": 44}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 3, "created_at": "2003-06-14T10:39:40+00:00", "updated_at": "2003-12-03T21:21:30+00:00", "name": "Yuki", "title": "Miss", "age": 50, "email": "vacuum2027@yahoo.com", "telephone": "1-321-809-2061", "gender": "Female", "language": "Armenian", "academic_degree": "Bachelor", "nationality": "Swiss", "occupation": "Valuer", "height": "1.84", "blood_type": "O\u2212", "weight": 71}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 4, "created_at": "2001-09-30T00:05:46+00:00", "updated_at": "2006-09-16T14:55:33+00:00", "name": "Fred", "title": "MMath", "age": 47, "email": "causes1859@outlook.com", "telephone": "(827) 127-3811", "gender": "Female", "language": "Assamese", "academic_degree": "PhD", "nationality": "Russian", "occupation": "Turkey Farmer", "height": "1.80", "blood_type": "A+", "weight": 39}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 5, "created_at": "2012-12-27T21:40:00+00:00", "updated_at": "2015-06-08T23:20:45+00:00", "name": "Emmitt", "title": "DPhil", "age": 39, "email": "athens1899@gmail.com", "telephone": "(470) 656-8003", "gender": "Other", "language": "English", "academic_degree": "Bachelor", "nationality": "Jordanian", "occupation": "Stone Sawyer", "height": "1.52", "blood_type": "A+", "weight": 82}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 6, "created_at": "2002-04-30T18:14:15+00:00", "updated_at": "2004-09-15T02:05:20+00:00", "name": "Hollis", "title": "MSc", "age": 52, "email": "fisheries1881@yandex.com", "telephone": "(519) 606-9896", "gender": "Other", "language": "Swati", "academic_degree": "Master", "nationality": "Chilean", "occupation": "Writer", "height": "1.85", "blood_type": "AB\u2212", "weight": 85}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 7, "created_at": "2003-09-11T17:13:51+00:00", "updated_at": "2016-08-04T09:35:18+00:00", "name": "Kip", "title": "M.D.", "age": 31, "email": "numbers1983@example.com", "telephone": "346-013-2638", "gender": "Other", "language": "Armenian", "academic_degree": "Master", "nationality": "Swiss", "occupation": "Salesman", "height": "1.89", "blood_type": "O+", "weight": 48}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 8, "created_at": "2012-06-19T07:18:11+00:00", "updated_at": "2017-10-10T14:05:38+00:00", "name": "Carie", "title": "Madam", "age": 49, "email": "watershed1819@example.com", "telephone": "(348) 881-9607", "gender": "Male", "language": "Kyrgyz", "academic_degree": "PhD", "nationality": "Guatemalan", "occupation": "Park Ranger", "height": "1.50", "blood_type": "O+", "weight": 83}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 9, "created_at": "2002-11-25T04:56:09+00:00", "updated_at": "2005-01-20T21:16:30+00:00", "name": "Steven", "title": "Mr.", "age": 54, "email": "llp1893@yahoo.com", "telephone": "830.247.8156", "gender": "Fluid", "language": "Catalan", "academic_degree": "Bachelor", "nationality": "Egyptian", "occupation": "Ambulance Driver", "height": "1.52", "blood_type": "AB+", "weight": 81}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 10, "created_at": "2001-02-23T17:43:25+00:00", "updated_at": "2022-09-09T16:51:15+00:00", "name": "Lore", "title": "Madam", "age": 61, "email": "resident2075@example.com", "telephone": "321.233.0702", "gender": "Female", "language": "Polish", "academic_degree": "Master", "nationality": "French", "occupation": "Registrar", "height": "1.99", "blood_type": "B+", "weight": 56}, "emitted_at": 1669830193008} -{"stream": "purchases", "data": {"id": 1, "product_id": 98, "user_id": 1, "added_to_cart_at": "2019-01-17T18:57:58+00:00", "purchased_at": "2020-06-30T18:57:58+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 2, "product_id": 39, "user_id": 2, "added_to_cart_at": "2019-06-02T19:57:09+00:00", "purchased_at": "2022-09-08T19:57:09+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 3, "product_id": 37, "user_id": 3, "added_to_cart_at": "2006-08-01T10:39:40+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 4, "product_id": 80, "user_id": 3, "added_to_cart_at": "2021-05-18T10:39:40+00:00", "purchased_at": "2022-11-14T10:39:40+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 5, "product_id": 40, "user_id": 4, "added_to_cart_at": "2003-12-18T00:05:46+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 6, "product_id": 88, "user_id": 4, "added_to_cart_at": "2009-02-24T00:05:46+00:00", "purchased_at": "2021-09-14T00:05:46+00:00", "returned_at": "2022-03-14T00:05:46+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 7, "product_id": 79, "user_id": 5, "added_to_cart_at": "2020-03-03T21:40:00+00:00", "purchased_at": "2022-11-17T21:40:00+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 8, "product_id": 67, "user_id": 6, "added_to_cart_at": "2008-03-02T18:14:15+00:00", "purchased_at": "2020-06-21T18:14:15+00:00", "returned_at": "2020-09-24T18:14:15+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 9, "product_id": 91, "user_id": 7, "added_to_cart_at": "2022-03-12T17:13:51+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 10, "product_id": 79, "user_id": 8, "added_to_cart_at": "2017-12-31T07:18:11+00:00", "purchased_at": "2019-05-14T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 11, "product_id": 91, "user_id": 8, "added_to_cart_at": "2022-03-24T07:18:11+00:00", "purchased_at": "2022-06-29T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 12, "product_id": 19, "user_id": 9, "added_to_cart_at": "2020-11-29T04:56:09+00:00", "purchased_at": "2022-03-02T04:56:09+00:00", "returned_at": "2022-04-12T04:56:09+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 13, "product_id": 63, "user_id": 10, "added_to_cart_at": "2003-08-05T17:43:25+00:00", "purchased_at": "2015-12-15T17:43:25+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1669830193015} +{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Rudolf", "title": "M.Des", "age": 66, "email": "wisconsin1930+1@yandex.com", "telephone": "(483) 676-2851", "gender": "Fluid", "language": "Arabic", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Orville", "title": "Miss", "age": 30, "email": "recipes2070+2@yahoo.com", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Rachell", "title": "M.A.", "age": 21, "email": "assets1924+3@protonmail.com", "telephone": "+1-(118)-374-3865", "gender": "Female", "language": "Dutch", "academic_degree": "PhD", "nationality": "Danish", "occupation": "Aeronautical Engineer", "height": "1.89", "blood_type": "AB+", "weight": 63}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 4, "created_at": "2000-09-08T14:31:35+00:00", "updated_at": "2011-04-22T07:48:29+00:00", "name": "Yer", "title": "M.Sc.Tech.", "age": 24, "email": "necessary2035+4@example.org", "telephone": "+1-(294)-359-4840", "gender": "Fluid", "language": "Malay", "academic_degree": "PhD", "nationality": "Argentinian", "occupation": "Line Manager", "height": "1.82", "blood_type": "B+", "weight": 43}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 5, "created_at": "2005-11-24T09:07:47+00:00", "updated_at": "2009-01-14T17:59:41+00:00", "name": "Alton", "title": "Miss", "age": 31, "email": "implementing1836+5@example.org", "telephone": "(712) 129-6627", "gender": "Other", "language": "Maltese", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Paint Consultant", "height": "1.69", "blood_type": "B\u2212", "weight": 88}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 6, "created_at": "2021-04-10T09:37:56+00:00", "updated_at": "2022-09-30T13:32:53+00:00", "name": "Octavio", "title": "Mr.", "age": 41, "email": "ind1929+6@yahoo.com", "telephone": "717-652-9752", "gender": "Male", "language": "Kurdish", "academic_degree": "PhD", "nationality": "Polish", "occupation": "Pathologist", "height": "1.83", "blood_type": "B\u2212", "weight": 41}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 7, "created_at": "2012-07-22T05:23:35+00:00", "updated_at": "2016-07-19T01:30:15+00:00", "name": "Casimira", "title": "B.Sc", "age": 63, "email": "coupled1824+7@live.com", "telephone": "1-515-852-9488", "gender": "Male", "language": "Khmer", "academic_degree": "Bachelor", "nationality": "Finnish", "occupation": "Chicken Chaser", "height": "1.60", "blood_type": "B\u2212", "weight": 75}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 8, "created_at": "2016-02-25T05:33:53+00:00", "updated_at": "2022-11-24T11:05:28+00:00", "name": "Terrell", "title": "Miss", "age": 55, "email": "stick1999+8@yahoo.com", "telephone": "023-973-2689", "gender": "Fluid", "language": "Tamil", "academic_degree": "PhD", "nationality": "Japanese", "occupation": "Trout Farmer", "height": "1.62", "blood_type": "O+", "weight": 43}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 9, "created_at": "2011-08-24T00:30:02+00:00", "updated_at": "2022-10-19T18:25:41+00:00", "name": "Ira", "title": "M.A.", "age": 37, "email": "efforts2075+9@yandex.com", "telephone": "(286) 981-5100", "gender": "Female", "language": "Croatian", "academic_degree": "Master", "nationality": "Cameroonian", "occupation": "Purchase Clerk", "height": "1.54", "blood_type": "AB+", "weight": 74}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 10, "created_at": "2005-09-08T00:49:12+00:00", "updated_at": "2017-04-13T16:22:54+00:00", "name": "Randall", "title": "Mrs.", "age": 34, "email": "intellectual1951+10@gmail.com", "telephone": "018-029-4112", "gender": "Male", "language": "Luxembourgish", "academic_degree": "Master", "nationality": "Mexican", "occupation": "Nursery Nurse", "height": "1.78", "blood_type": "AB\u2212", "weight": 58}, "emitted_at": 1671748915185} +{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2003-02-23T11:53:10+00:00", "purchased_at": "2011-03-30T11:53:10+00:00", "returned_at": null}, "emitted_at": 1671748915424} +{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "added_to_cart_at": "2019-12-02T14:32:17+00:00", "purchased_at": "2021-02-08T14:32:17+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 3, "product_id": 5, "user_id": 3, "added_to_cart_at": "2014-01-25T14:59:20+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 4, "product_id": 21, "user_id": 4, "added_to_cart_at": "2016-11-27T05:20:11+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 5, "product_id": 51, "user_id": 5, "added_to_cart_at": "2020-10-15T11:54:28+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 6, "product_id": 66, "user_id": 6, "added_to_cart_at": "2017-02-24T14:41:33+00:00", "purchased_at": "2022-02-20T14:41:33+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 7, "product_id": 35, "user_id": 8, "added_to_cart_at": "2021-12-01T17:46:29+00:00", "purchased_at": "2021-12-15T17:46:29+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 8, "product_id": 60, "user_id": 9, "added_to_cart_at": "2021-03-07T15:19:19+00:00", "purchased_at": "2022-05-14T15:19:19+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 9, "product_id": 22, "user_id": 9, "added_to_cart_at": "2021-08-30T23:13:31+00:00", "purchased_at": "2021-12-28T23:13:31+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 10, "product_id": 4, "user_id": 10, "added_to_cart_at": "2008-03-28T03:03:36+00:00", "purchased_at": "2012-02-18T03:03:36+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1671748915434} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py new file mode 100644 index 000000000000..be338ddb4afe --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.models import AirbyteMessage + + +class AirbyteMessageWithCachedJSON(AirbyteMessage): + """ + I a monkeypatch to AirbyteMessage which pre-renders the JSON-representation of the object upon initialization. + This allows the JSON to be calculated in the process that builds the object rather than the main process. + + Note: We can't use @cache here because the LRU cache is not serializable when passed to child workers. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._json = self.json(exclude_unset=True) + self.json = self.get_json + + def get_json(self, **kwargs): + return self._json diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py new file mode 100644 index 000000000000..53cf05be4177 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from multiprocessing import current_process +from typing import Dict + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Numeric + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class PurchaseGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters + """ + + seed_with_offset = self.seed + if self.seed is not None and len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global dt + global numeric + + dt = Datetime(seed=seed_with_offset) + numeric = Numeric(seed=seed_with_offset) + + def random_date_in_range( + self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() + ) -> datetime.datetime: + time_between_dates = end_date - start_date + days_between_dates = time_between_dates.days + if days_between_dates < 2: + days_between_dates = 2 + random_number_of_days = numeric.integer_number(0, days_between_dates) + random_date = start_date + datetime.timedelta(days=random_number_of_days) + return random_date + + def generate(self, user_id: int) -> list[Dict]: + """ + Because we are doing this work in parallel processes, we need a deterministic way to know what a purchase's ID should be given on the input of a user_id. + tldr; Every 10 user_ids produce 10 purchases. User ID x5 has no purchases, User ID mod x7 has 2, and everyone else has 1 + """ + + purchases: list[Dict] = [] + last_user_id_digit = int(repr(user_id)[-1]) + purchase_count = 1 + id_offset = 0 + if last_user_id_digit - 1 == 5: + purchase_count = 0 + elif last_user_id_digit - 1 == 6: + id_offset = 1 + elif last_user_id_digit - 1 == 7: + id_offset = 1 + purchase_count = 2 + + total_products = 100 + i = 0 + + while purchase_count > 0: + id = user_id + i + 1 - id_offset + time_a = dt.datetime() + time_b = dt.datetime() + created_at = time_a if time_a <= time_b else time_b + product_id = numeric.integer_number(1, total_products) + added_to_cart_at = self.random_date_in_range(created_at) + purchased_at = ( + self.random_date_in_range(added_to_cart_at) + if added_to_cart_at is not None and numeric.integer_number(1, 100) <= 70 + else None + ) # 70% likely to purchase the item in the cart + returned_at = ( + self.random_date_in_range(purchased_at) if purchased_at is not None and numeric.integer_number(1, 100) <= 15 else None + ) # 15% likely to return the item + + purchase = { + "id": id, + "product_id": product_id, + "user_id": user_id + 1, + "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, + "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, + "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, + } + + record = AirbyteRecordMessage(stream=self.stream_name, data=purchase, emitted_at=now_millis()) + message = AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) + purchases.append(message) + + purchase_count = purchase_count - 1 + i += 1 + + return purchases diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index 25a35776e34e..3772cea75516 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -19,14 +19,14 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, "Count option is missing" def streams(self, config: Mapping[str, Any]) -> List[Stream]: - count: int = config["count"] if "count" in config else 0 seed: int = config["seed"] if "seed" in config else None records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500 records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100 + parallelism: int = config["parallelism"] if "parallelism" in config else 4 return [ - Products(count, seed, records_per_sync, records_per_slice), - Users(count, seed, records_per_sync, records_per_slice), - Purchases(count, seed, records_per_sync, records_per_slice), + Products(count, seed, parallelism, records_per_sync, records_per_slice), + Users(count, seed, parallelism, records_per_sync, records_per_slice), + Purchases(count, seed, parallelism, records_per_sync, records_per_slice), ] diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index 1018a957f109..0d20f791000d 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -35,8 +35,16 @@ "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", "type": "integer", "minimum": 1, - "default": 100, + "default": 1000, "order": 3 + }, + "parallelism": { + "title": "Parallelism", + "description": "How many parallel workers should we use to generate fake data? Choose a value equal to the number of CPUs you will allocate to this source.", + "type": "integer", + "minimum": 1, + "default": 4, + "order": 4 } } } diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index eab4969305f5..a25e93813615 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -2,23 +2,22 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import datetime import os +from multiprocessing import Pool from typing import Any, Dict, Iterable, Mapping, Optional -from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from mimesis import Datetime, Numeric, Person -from mimesis.locales import Locale -from .utils import format_airbyte_time, read_json +from .purchase_generator import PurchaseGenerator +from .user_generator import UserGenerator +from .utils import generate_estimate, read_json class Products(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.seed = seed self.records_per_sync = records_per_sync @@ -64,14 +63,14 @@ class Users(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.person = Person(locale=Locale.EN, seed=self.seed) - self.dt = Datetime(seed=self.seed) + self.parallelism = parallelism + self.generator = UserGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -88,58 +87,33 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def generate_user(self, user_id: int): - time_a = self.dt.datetime() - time_b = self.dt.datetime() - - profile = { - "id": user_id + 1, - "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), - "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), - "name": self.person.name(), - "title": self.person.title(), - "age": self.person.age(), - "email": self.person.email(), - "telephone": self.person.telephone(), - "gender": self.person.gender(), - "language": self.person.language(), - "academic_degree": self.person.academic_degree(), - "nationality": self.person.nationality(), - "occupation": self.person.occupation(), - "height": self.person.height(), - "blood_type": self.person.blood_type(), - "weight": self.person.weight(), - } - - while not profile["created_at"]: - profile["created_at"] = format_airbyte_time(self.dt.datetime()) - - if not profile["updated_at"]: - profile["updated_at"] = profile["created_at"] + 1 - - return profile - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ + total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 records_in_sync = 0 - records_in_slice = 0 median_record_byte_size = 450 yield generate_estimate(self.name, self.count - total_records, median_record_byte_size) - for i in range(total_records, self.count): - user = self.generate_user(i) - yield user - total_records += 1 - records_in_sync += 1 - records_in_slice += 1 + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while records_in_sync < self.count and records_in_sync < self.records_per_sync: + records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) + if records_remaining_this_loop <= 0: + break + users = pool.map(self.generator.generate, range(total_records, total_records + records_remaining_this_loop)) + for user in users: + total_records += 1 + records_in_sync += 1 + yield user - if records_in_slice >= self.records_per_slice: - self.state = {self.cursor_field: total_records, "seed": self.seed} - records_in_slice = 0 + if records_in_sync >= self.records_per_sync: + break - if records_in_sync == self.records_per_sync: - break + self.state = {self.cursor_field: total_records, "seed": self.seed} self.state = {self.cursor_field: total_records, "seed": self.seed} @@ -148,14 +122,14 @@ class Purchases(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.dt = Datetime(seed=self.seed) - self.numeric = Numeric(seed=self.seed) + self.parallelism = parallelism + self.generator = PurchaseGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -172,88 +146,37 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def random_date_in_range( - self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() - ) -> datetime.datetime: - time_between_dates = end_date - start_date - days_between_dates = time_between_dates.days - if days_between_dates < 2: - days_between_dates = 2 - random_number_of_days = self.numeric.integer_number(0, days_between_dates) - random_date = start_date + datetime.timedelta(days=random_number_of_days) - return random_date - - def generate_purchases(self, user_id: int, purchases_count: int) -> list[Dict]: - purchases: list[Dict] = [] - purchase_percent_remaining = 70 # ~30% of people will have no purchases - total_products = 100 - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) - i = 0 - - time_a = self.dt.datetime() - time_b = self.dt.datetime() - created_at = time_a if time_a <= time_b else time_b - - while purchase_percent_remaining > 0: - id = purchases_count + i + 1 - product_id = self.numeric.integer_number(1, total_products) - added_to_cart_at = self.random_date_in_range(created_at) - purchased_at = ( - self.random_date_in_range(added_to_cart_at) - if added_to_cart_at is not None and self.numeric.integer_number(1, 100) <= 70 - else None - ) # 70% likely to purchase the item in the cart - returned_at = ( - self.random_date_in_range(purchased_at) if purchased_at is not None and self.numeric.integer_number(1, 100) <= 15 else None - ) # 15% likely to return the item - - purchase = { - "id": id, - "product_id": product_id, - "user_id": user_id, - "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, - "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, - "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, - } - purchases.append(purchase) - - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) - i += 1 - return purchases - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ + total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 total_user_records = self.state["user_id"] if "user_id" in self.state else 0 user_records_in_sync = 0 - user_records_in_slice = 0 + # a fuzzy guess, some users have purchases, some don't median_record_byte_size = 230 - yield generate_estimate( - self.name, (self.count - total_user_records) * 1.3, median_record_byte_size - ) # a fuzzy guess, some users have purchases, some don't - - for i in range(total_user_records, self.count): - purchases = self.generate_purchases(i + 1, total_purchase_records) - for purchase in purchases: - total_purchase_records += 1 - yield purchase - total_user_records += 1 - user_records_in_sync += 1 - user_records_in_slice += 1 - - if user_records_in_slice >= self.records_per_slice: - self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} - user_records_in_slice = 0 + yield generate_estimate(self.name, (self.count - total_user_records) * 1.3, median_record_byte_size) - if user_records_in_sync == self.records_per_sync: - break + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while total_user_records < self.count and user_records_in_sync < self.records_per_sync: + records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) + if records_remaining_this_loop <= 0: + break + carts = pool.map(self.generator.generate, range(total_user_records, total_user_records + records_remaining_this_loop)) + for purchases in carts: + for purchase in purchases: + total_purchase_records += 1 + yield purchase - self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + total_user_records += 1 + user_records_in_sync += 1 + if user_records_in_sync >= self.records_per_sync: + break -def generate_estimate(stream_name: str, total: int, bytes_per_row: int): - emitted_at = int(datetime.datetime.now().timestamp() * 1000) - estimate_message = AirbyteEstimateTraceMessage( - type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) - ) - return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py new file mode 100644 index 000000000000..3fcedd2b6ad4 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from multiprocessing import current_process + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Person +from mimesis.locales import Locale + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class UserGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters + """ + + seed_with_offset = self.seed + if self.seed is not None and len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global person + global dt + + person = Person(locale=Locale.EN, seed=seed_with_offset) + dt = Datetime(seed=seed_with_offset) + + def generate(self, user_id: int): + time_a = dt.datetime() + time_b = dt.datetime() + + # faker doesn't always produce unique email addresses, so to enforce uniqueness, we will append the user_id to the prefix + email_parts = person.email().split("@") + email = f"{email_parts[0]}+{user_id + 1}@{email_parts[1]}" + + profile = { + "id": user_id + 1, + "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), + "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), + "name": person.name(), + "title": person.title(), + "age": person.age(), + "email": email, + "telephone": person.telephone(), + "gender": person.gender(), + "language": person.language(), + "academic_degree": person.academic_degree(), + "nationality": person.nationality(), + "occupation": person.occupation(), + "height": person.height(), + "blood_type": person.blood_type(), + "weight": person.weight(), + } + + while not profile["created_at"]: + profile["created_at"] = format_airbyte_time(dt.datetime()) + + if not profile["updated_at"]: + profile["updated_at"] = profile["created_at"] + 1 + + record = AirbyteRecordMessage(stream=self.stream_name, data=profile, emitted_at=now_millis()) + return AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/utils.py b/airbyte-integrations/connectors/source-faker/source_faker/utils.py index 12970853c956..9c094b11bcb7 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/utils.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/utils.py @@ -5,6 +5,8 @@ import datetime import json +from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType + def read_json(filepath): with open(filepath, "r") as f: @@ -17,3 +19,15 @@ def format_airbyte_time(d: datetime): s = s.replace(" ", "T") s += "+00:00" return s + + +def now_millis(): + return int(datetime.datetime.now().timestamp() * 1000) + + +def generate_estimate(stream_name: str, total: int, bytes_per_row: int): + emitted_at = int(datetime.datetime.now().timestamp() * 1000) + estimate_message = AirbyteEstimateTraceMessage( + type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) + ) + return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index 8a989b7778ac..193960b73453 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -25,7 +25,7 @@ def exception(a,b,**kwargs): def schemas_are_valid(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -36,7 +36,7 @@ def schemas_are_valid(): def test_source_streams(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -64,7 +64,7 @@ def test_source_streams(): def test_read_small_random_data(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -98,7 +98,7 @@ def test_read_small_random_data(): def test_no_read_limit_hit(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -128,7 +128,7 @@ def test_no_read_limit_hit(): def test_read_big_random_data(): source = SourceFaker() - config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000} + config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -163,7 +163,7 @@ def test_read_big_random_data(): def test_with_purchases(): source = SourceFaker() - config = {"count": 1000, "records_per_sync": 1000} + config = {"count": 1000, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -205,7 +205,7 @@ def test_with_purchases(): def test_sync_ends_with_limit(): source = SourceFaker() - config = {"count": 100, "records_per_sync": 5} + config = {"count": 100, "records_per_sync": 5, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -239,7 +239,7 @@ def test_read_with_seed(): """ source = SourceFaker() - config = {"count": 1, "seed": 100} + config = {"count": 1, "seed": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -253,14 +253,14 @@ def test_read_with_seed(): iterator = source.read(logger, config, catalog, state) records = [row for row in iterator if row.type is Type.RECORD] - assert records[0].record.data["occupation"] == "Roadworker" - assert records[0].record.data["email"] == "reproduce1856@outlook.com" + assert records[0].record.data["occupation"] == "Cartoonist" + assert records[0].record.data["email"] == "reflect1958+1@yahoo.com" def test_ensure_no_purchases_without_users(): with pytest.raises(ValueError): source = SourceFaker() - config = {"count": 100} + config = {"count": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ {"stream": {"name": "purchases", "json_schema": {}}, "sync_mode": "incremental", "destination_sync_mode": "overwrite"}, From 9ad7e04487f5ab364e6768562032c234c35954e3 Mon Sep 17 00:00:00 2001 From: evantahler Date: Mon, 2 Jan 2023 14:35:10 -0800 Subject: [PATCH 10/12] update readme --- docs/integrations/sources/faker.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index 337434ce4b1c..ef69a14377cb 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -81,18 +81,18 @@ None! ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | -| 2.0.0 | 2022-12-14 | [20492](https://github.com/airbytehq/airbyte/pull/20492) | Decouple stream states for better parallelism | -| 1.0.0 | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490) | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename | -| 0.2.1 | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197) | Emit `AirbyteEstimateTraceMessage` | -| 0.2.0 | 2022-10-14 | [18021](https://github.com/airbytehq/airbyte/pull/18021) | Move to mimesis for speed! | -| 0.1.8 | 2022-10-12 | [17889](https://github.com/airbytehq/airbyte/pull/17889) | Bump to test publish command (2) | -| 0.1.7 | 2022-10-11 | [17848](https://github.com/airbytehq/airbyte/pull/17848) | Bump to test publish command | -| 0.1.6 | 2022-09-07 | [16418](https://github.com/airbytehq/airbyte/pull/16418) | Log start of each stream | -| 0.1.5 | 2022-06-10 | [13695](https://github.com/airbytehq/airbyte/pull/13695) | Emit timestamps in the proper ISO format | -| 0.1.4 | 2022-05-27 | [13298](https://github.com/airbytehq/airbyte/pull/13298) | Test publication flow | -| 0.1.3 | 2022-05-27 | [13248](https://github.com/airbytehq/airbyte/pull/13248) | Add options for records_per_sync and page_size | -| 0.1.2 | 2022-05-26 | [13248](https://github.com/airbytehq/airbyte/pull/13293) | Test publication flow | -| 0.1.1 | 2022-05-26 | [13235](https://github.com/airbytehq/airbyte/pull/13235) | Publish for AMD and ARM (M1 Macs) & remove User.birthdate | -| 0.1.0 | 2022-04-12 | [11738](https://github.com/airbytehq/airbyte/pull/11738) | The Faker Source is created | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 2.0.0 | 2022-12-14 | [20492](https://github.com/airbytehq/airbyte/pull/20492) and [20741](https://github.com/airbytehq/airbyte/pull/20741) | Decouple stream states for better parallelism | +| 1.0.0 | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490) | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename | +| 0.2.1 | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197) | Emit `AirbyteEstimateTraceMessage` | +| 0.2.0 | 2022-10-14 | [18021](https://github.com/airbytehq/airbyte/pull/18021) | Move to mimesis for speed! | +| 0.1.8 | 2022-10-12 | [17889](https://github.com/airbytehq/airbyte/pull/17889) | Bump to test publish command (2) | +| 0.1.7 | 2022-10-11 | [17848](https://github.com/airbytehq/airbyte/pull/17848) | Bump to test publish command | +| 0.1.6 | 2022-09-07 | [16418](https://github.com/airbytehq/airbyte/pull/16418) | Log start of each stream | +| 0.1.5 | 2022-06-10 | [13695](https://github.com/airbytehq/airbyte/pull/13695) | Emit timestamps in the proper ISO format | +| 0.1.4 | 2022-05-27 | [13298](https://github.com/airbytehq/airbyte/pull/13298) | Test publication flow | +| 0.1.3 | 2022-05-27 | [13248](https://github.com/airbytehq/airbyte/pull/13248) | Add options for records_per_sync and page_size | +| 0.1.2 | 2022-05-26 | [13248](https://github.com/airbytehq/airbyte/pull/13293) | Test publication flow | +| 0.1.1 | 2022-05-26 | [13235](https://github.com/airbytehq/airbyte/pull/13235) | Publish for AMD and ARM (M1 Macs) & remove User.birthdate | +| 0.1.0 | 2022-04-12 | [11738](https://github.com/airbytehq/airbyte/pull/11738) | The Faker Source is created | From 32c3773595c702dd7c5d7e3c35f8dae5f06f4d11 Mon Sep 17 00:00:00 2001 From: evantahler Date: Mon, 2 Jan 2023 14:42:25 -0800 Subject: [PATCH 11/12] update expected records --- .../integration_tests/expected_records.jsonl | 240 +++++++++--------- 1 file changed, 120 insertions(+), 120 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index e6e1c3e60c9a..685c3769f6a4 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -1,120 +1,120 @@ -{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Rudolf", "title": "M.Des", "age": 66, "email": "wisconsin1930+1@yandex.com", "telephone": "(483) 676-2851", "gender": "Fluid", "language": "Arabic", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1671748915184} -{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Orville", "title": "Miss", "age": 30, "email": "recipes2070+2@yahoo.com", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1671748915184} -{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Rachell", "title": "M.A.", "age": 21, "email": "assets1924+3@protonmail.com", "telephone": "+1-(118)-374-3865", "gender": "Female", "language": "Dutch", "academic_degree": "PhD", "nationality": "Danish", "occupation": "Aeronautical Engineer", "height": "1.89", "blood_type": "AB+", "weight": 63}, "emitted_at": 1671748915184} -{"stream": "users", "data": {"id": 4, "created_at": "2000-09-08T14:31:35+00:00", "updated_at": "2011-04-22T07:48:29+00:00", "name": "Yer", "title": "M.Sc.Tech.", "age": 24, "email": "necessary2035+4@example.org", "telephone": "+1-(294)-359-4840", "gender": "Fluid", "language": "Malay", "academic_degree": "PhD", "nationality": "Argentinian", "occupation": "Line Manager", "height": "1.82", "blood_type": "B+", "weight": 43}, "emitted_at": 1671748915184} -{"stream": "users", "data": {"id": 5, "created_at": "2005-11-24T09:07:47+00:00", "updated_at": "2009-01-14T17:59:41+00:00", "name": "Alton", "title": "Miss", "age": 31, "email": "implementing1836+5@example.org", "telephone": "(712) 129-6627", "gender": "Other", "language": "Maltese", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Paint Consultant", "height": "1.69", "blood_type": "B\u2212", "weight": 88}, "emitted_at": 1671748915185} -{"stream": "users", "data": {"id": 6, "created_at": "2021-04-10T09:37:56+00:00", "updated_at": "2022-09-30T13:32:53+00:00", "name": "Octavio", "title": "Mr.", "age": 41, "email": "ind1929+6@yahoo.com", "telephone": "717-652-9752", "gender": "Male", "language": "Kurdish", "academic_degree": "PhD", "nationality": "Polish", "occupation": "Pathologist", "height": "1.83", "blood_type": "B\u2212", "weight": 41}, "emitted_at": 1671748915185} -{"stream": "users", "data": {"id": 7, "created_at": "2012-07-22T05:23:35+00:00", "updated_at": "2016-07-19T01:30:15+00:00", "name": "Casimira", "title": "B.Sc", "age": 63, "email": "coupled1824+7@live.com", "telephone": "1-515-852-9488", "gender": "Male", "language": "Khmer", "academic_degree": "Bachelor", "nationality": "Finnish", "occupation": "Chicken Chaser", "height": "1.60", "blood_type": "B\u2212", "weight": 75}, "emitted_at": 1671748915185} -{"stream": "users", "data": {"id": 8, "created_at": "2016-02-25T05:33:53+00:00", "updated_at": "2022-11-24T11:05:28+00:00", "name": "Terrell", "title": "Miss", "age": 55, "email": "stick1999+8@yahoo.com", "telephone": "023-973-2689", "gender": "Fluid", "language": "Tamil", "academic_degree": "PhD", "nationality": "Japanese", "occupation": "Trout Farmer", "height": "1.62", "blood_type": "O+", "weight": 43}, "emitted_at": 1671748915185} -{"stream": "users", "data": {"id": 9, "created_at": "2011-08-24T00:30:02+00:00", "updated_at": "2022-10-19T18:25:41+00:00", "name": "Ira", "title": "M.A.", "age": 37, "email": "efforts2075+9@yandex.com", "telephone": "(286) 981-5100", "gender": "Female", "language": "Croatian", "academic_degree": "Master", "nationality": "Cameroonian", "occupation": "Purchase Clerk", "height": "1.54", "blood_type": "AB+", "weight": 74}, "emitted_at": 1671748915185} -{"stream": "users", "data": {"id": 10, "created_at": "2005-09-08T00:49:12+00:00", "updated_at": "2017-04-13T16:22:54+00:00", "name": "Randall", "title": "Mrs.", "age": 34, "email": "intellectual1951+10@gmail.com", "telephone": "018-029-4112", "gender": "Male", "language": "Luxembourgish", "academic_degree": "Master", "nationality": "Mexican", "occupation": "Nursery Nurse", "height": "1.78", "blood_type": "AB\u2212", "weight": 58}, "emitted_at": 1671748915185} -{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2003-02-23T11:53:10+00:00", "purchased_at": "2011-03-30T11:53:10+00:00", "returned_at": null}, "emitted_at": 1671748915424} -{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "added_to_cart_at": "2019-12-02T14:32:17+00:00", "purchased_at": "2021-02-08T14:32:17+00:00", "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 3, "product_id": 5, "user_id": 3, "added_to_cart_at": "2014-01-25T14:59:20+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 4, "product_id": 21, "user_id": 4, "added_to_cart_at": "2016-11-27T05:20:11+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 5, "product_id": 51, "user_id": 5, "added_to_cart_at": "2020-10-15T11:54:28+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 6, "product_id": 66, "user_id": 6, "added_to_cart_at": "2017-02-24T14:41:33+00:00", "purchased_at": "2022-02-20T14:41:33+00:00", "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 7, "product_id": 35, "user_id": 8, "added_to_cart_at": "2021-12-01T17:46:29+00:00", "purchased_at": "2021-12-15T17:46:29+00:00", "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 8, "product_id": 60, "user_id": 9, "added_to_cart_at": "2021-03-07T15:19:19+00:00", "purchased_at": "2022-05-14T15:19:19+00:00", "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 9, "product_id": 22, "user_id": 9, "added_to_cart_at": "2021-08-30T23:13:31+00:00", "purchased_at": "2021-12-28T23:13:31+00:00", "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "purchases", "data": {"id": 10, "product_id": 4, "user_id": 10, "added_to_cart_at": "2008-03-28T03:03:36+00:00", "purchased_at": "2012-02-18T03:03:36+00:00", "returned_at": null}, "emitted_at": 1671748915425} -{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1671748915430} -{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1671748915431} -{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1671748915432} -{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1671748915433} -{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1671748915434} -{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1671748915434} +{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Rudolf", "title": "M.Des", "age": 66, "email": "wisconsin1930+1@yandex.com", "telephone": "(483) 676-2851", "gender": "Fluid", "language": "Arabic", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1672699162606} +{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Orville", "title": "Miss", "age": 30, "email": "recipes2070+2@yahoo.com", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1672699162606} +{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Rachell", "title": "M.A.", "age": 21, "email": "assets1924+3@protonmail.com", "telephone": "+1-(118)-374-3865", "gender": "Female", "language": "Dutch", "academic_degree": "PhD", "nationality": "Danish", "occupation": "Aeronautical Engineer", "height": "1.89", "blood_type": "AB+", "weight": 63}, "emitted_at": 1672699162606} +{"stream": "users", "data": {"id": 4, "created_at": "2015-09-08T11:14:43+00:00", "updated_at": "2023-01-17T07:48:28+00:00", "name": "Yer", "title": "M.Sc.Tech.", "age": 24, "email": "necessary2035+4@example.org", "telephone": "+1-(294)-359-4840", "gender": "Fluid", "language": "Malay", "academic_degree": "PhD", "nationality": "Argentinian", "occupation": "Line Manager", "height": "1.82", "blood_type": "B+", "weight": 43}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 5, "created_at": "2014-05-30T00:26:53+00:00", "updated_at": "2017-11-04T05:40:46+00:00", "name": "Alton", "title": "Miss", "age": 31, "email": "implementing1836+5@example.org", "telephone": "(712) 129-6627", "gender": "Other", "language": "Maltese", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Paint Consultant", "height": "1.69", "blood_type": "B\u2212", "weight": 88}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 6, "created_at": "2009-02-24T10:57:46+00:00", "updated_at": "2016-07-17T21:12:19+00:00", "name": "Octavio", "title": "Mr.", "age": 41, "email": "ind1929+6@yahoo.com", "telephone": "717-652-9752", "gender": "Male", "language": "Kurdish", "academic_degree": "PhD", "nationality": "Polish", "occupation": "Pathologist", "height": "1.83", "blood_type": "B\u2212", "weight": 41}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 7, "created_at": "2001-08-08T23:51:25+00:00", "updated_at": "2018-08-28T16:25:37+00:00", "name": "Casimira", "title": "B.Sc", "age": 63, "email": "coupled1824+7@live.com", "telephone": "1-515-852-9488", "gender": "Male", "language": "Khmer", "academic_degree": "Bachelor", "nationality": "Finnish", "occupation": "Chicken Chaser", "height": "1.60", "blood_type": "B\u2212", "weight": 75}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 8, "created_at": "2021-03-12T17:56:44+00:00", "updated_at": "2021-12-12T02:28:42+00:00", "name": "Terrell", "title": "Miss", "age": 55, "email": "stick1999+8@yahoo.com", "telephone": "023-973-2689", "gender": "Fluid", "language": "Tamil", "academic_degree": "PhD", "nationality": "Japanese", "occupation": "Trout Farmer", "height": "1.62", "blood_type": "O+", "weight": 43}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 9, "created_at": "2000-08-02T09:45:54+00:00", "updated_at": "2003-03-17T12:23:31+00:00", "name": "Ira", "title": "M.A.", "age": 37, "email": "efforts2075+9@yandex.com", "telephone": "(286) 981-5100", "gender": "Female", "language": "Croatian", "academic_degree": "Master", "nationality": "Cameroonian", "occupation": "Purchase Clerk", "height": "1.54", "blood_type": "AB+", "weight": 74}, "emitted_at": 1672699162607} +{"stream": "users", "data": {"id": 10, "created_at": "2007-01-25T06:34:58+00:00", "updated_at": "2018-10-13T20:10:10+00:00", "name": "Randall", "title": "Mrs.", "age": 34, "email": "intellectual1951+10@gmail.com", "telephone": "018-029-4112", "gender": "Male", "language": "Luxembourgish", "academic_degree": "Master", "nationality": "Mexican", "occupation": "Nursery Nurse", "height": "1.78", "blood_type": "AB\u2212", "weight": 58}, "emitted_at": 1672699162607} +{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2003-02-23T11:53:10+00:00", "purchased_at": "2011-03-30T11:53:10+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "added_to_cart_at": "2022-08-08T13:40:25+00:00", "purchased_at": "2022-10-11T13:40:25+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 3, "product_id": 28, "user_id": 3, "added_to_cart_at": "2021-09-06T00:23:29+00:00", "purchased_at": "2022-06-30T00:23:29+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 4, "product_id": 21, "user_id": 4, "added_to_cart_at": "2016-11-27T05:20:11+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 5, "product_id": 51, "user_id": 5, "added_to_cart_at": "2020-02-14T11:54:28+00:00", "purchased_at": "2022-08-13T11:54:28+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 6, "product_id": 35, "user_id": 6, "added_to_cart_at": "2013-05-20T14:41:33+00:00", "purchased_at": "2017-06-19T14:41:33+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 7, "product_id": 41, "user_id": 8, "added_to_cart_at": "2021-12-28T17:46:29+00:00", "purchased_at": "2022-08-01T17:46:29+00:00", "returned_at": null}, "emitted_at": 1672699162839} +{"stream": "purchases", "data": {"id": 8, "product_id": 22, "user_id": 9, "added_to_cart_at": "2022-07-15T15:19:19+00:00", "purchased_at": "2022-09-13T15:19:19+00:00", "returned_at": null}, "emitted_at": 1672699162840} +{"stream": "purchases", "data": {"id": 9, "product_id": 4, "user_id": 9, "added_to_cart_at": "2019-07-06T23:13:31+00:00", "purchased_at": "2020-06-25T23:13:31+00:00", "returned_at": null}, "emitted_at": 1672699162840} +{"stream": "purchases", "data": {"id": 10, "product_id": 66, "user_id": 10, "added_to_cart_at": "2017-08-09T02:50:52+00:00", "purchased_at": "2020-06-26T02:50:52+00:00", "returned_at": null}, "emitted_at": 1672699162840} +{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1672699162843} +{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1672699162844} +{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1672699162845} +{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1672699162846} +{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1672699162847} +{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1672699162847} From 560495391eec0e243409d0fd05c405aa233e0f3a Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 2 Jan 2023 23:17:32 +0000 Subject: [PATCH 12/12] auto-bump connector version --- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 26c4c500ed4a..7a56685403a4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -467,7 +467,7 @@ - name: Sample Data (Faker) sourceDefinitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 dockerRepository: airbyte/source-faker - dockerImageTag: 1.0.0 + dockerImageTag: 2.0.0 documentationUrl: https://docs.airbyte.com/integrations/sources/faker sourceType: api releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e45e96420691..f8efa0921ef6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3866,7 +3866,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-faker:1.0.0" +- dockerImage: "airbyte/source-faker:2.0.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/faker" connectionSpecification: @@ -3907,8 +3907,17 @@ \ before a state message is emitted?" type: "integer" minimum: 1 - default: 100 + default: 1000 order: 3 + parallelism: + title: "Parallelism" + description: "How many parallel workers should we use to generate fake data?\ + \ Choose a value equal to the number of CPUs you will allocate to this\ + \ source." + type: "integer" + minimum: 1 + default: 4 + order: 4 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: []