From abf9b49a2ad22fcbf3f4d1be5f2636285f744f4c Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 20 Dec 2022 15:50:30 -0800 Subject: [PATCH 01/13] Faker is 250% faster --- .../sources/utils/record_helper.py | 4 +- .../source-faker/source_faker/source.py | 8 +- .../source-faker/source_faker/streams.py | 185 +++++++++++------- .../source-faker/unit_tests/unit_test.py | 24 +-- 4 files changed, 135 insertions(+), 86 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py index 482596230f04..71693ca73a13 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py @@ -20,7 +20,9 @@ def stream_data_to_airbyte_message( if schema is None: schema = {} - if isinstance(data_or_message, dict): + if isinstance(data_or_message, AirbyteMessage): + return data_or_message + elif isinstance(data_or_message, dict): data = data_or_message now_millis = int(datetime.datetime.now().timestamp() * 1000) # Transform object fields according to config. Most likely you will diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index 25a35776e34e..a8b877951861 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -19,14 +19,14 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, "Count option is missing" def streams(self, config: Mapping[str, Any]) -> List[Stream]: - count: int = config["count"] if "count" in config else 0 seed: int = config["seed"] if "seed" in config else None records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500 records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100 + threads: int = config["threads"] if "threads" in config else 4 return [ - Products(count, seed, records_per_sync, records_per_slice), - Users(count, seed, records_per_sync, records_per_slice), - Purchases(count, seed, records_per_sync, records_per_slice), + Products(count, seed, threads, records_per_sync, records_per_slice), + Users(count, seed, threads, records_per_sync, records_per_slice), + Purchases(count, seed, threads, records_per_sync, records_per_slice), ] diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index eab4969305f5..187f50c35b91 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -4,21 +4,38 @@ import datetime import os -from typing import Any, Dict, Iterable, Mapping, Optional -from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType +from multiprocessing import Pool, current_process +# from multiprocessing.pool import ThreadPool + +from typing import Any, Dict, Iterable, Mapping, Optional, List + +from airbyte_cdk.models import AirbyteMessage, AirbyteEstimateTraceMessage, Type, AirbyteTraceMessage, AirbyteRecordMessage, EstimateType, TraceType from airbyte_cdk.sources.streams import IncrementalMixin, Stream from mimesis import Datetime, Numeric, Person from mimesis.locales import Locale from .utils import format_airbyte_time, read_json +class FakerMultithreaded: + def worker_init(self): + """For the workers, we want a unique instance of the faker packages with their own seeds to differentiate the generated responses""" + global person + global dt + global numeric + seed_with_offset = self.seed + if self.seed != None: + seed_with_offset = self.seed + current_process()._identity[0] + person = Person(locale=Locale.EN, seed=seed_with_offset) + dt = Datetime(seed=seed_with_offset) + numeric = Numeric(seed=seed_with_offset) + class Products(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, threads:int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.seed = seed self.records_per_sync = records_per_sync @@ -60,18 +77,17 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.state = {self.cursor_field: total_records, "seed": self.seed} -class Users(Stream, IncrementalMixin): +class Users(Stream, IncrementalMixin, FakerMultithreaded): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, threads:int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.person = Person(locale=Locale.EN, seed=self.seed) - self.dt = Datetime(seed=self.seed) + self.threads = threads @property def state_checkpoint_interval(self) -> Optional[int]: @@ -89,66 +105,68 @@ def state(self, value: Mapping[str, Any]): self._state = value def generate_user(self, user_id: int): - time_a = self.dt.datetime() - time_b = self.dt.datetime() + time_a = dt.datetime() + time_b = dt.datetime() profile = { "id": user_id + 1, "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), - "name": self.person.name(), - "title": self.person.title(), - "age": self.person.age(), - "email": self.person.email(), - "telephone": self.person.telephone(), - "gender": self.person.gender(), - "language": self.person.language(), - "academic_degree": self.person.academic_degree(), - "nationality": self.person.nationality(), - "occupation": self.person.occupation(), - "height": self.person.height(), - "blood_type": self.person.blood_type(), - "weight": self.person.weight(), + "name": person.name(), + "title": person.title(), + "age": person.age(), + "email": person.email(), + "telephone": person.telephone(), + "gender": person.gender(), + "language": person.language(), + "academic_degree": person.academic_degree(), + "nationality": person.nationality(), + "occupation": person.occupation(), + "height": person.height(), + "blood_type": person.blood_type(), + "weight": person.weight(), } while not profile["created_at"]: - profile["created_at"] = format_airbyte_time(self.dt.datetime()) + profile["created_at"] = format_airbyte_time(dt.datetime()) if not profile["updated_at"]: profile["updated_at"] = profile["created_at"] + 1 - return profile + record = AirbyteRecordMessage(stream=self.name, data=profile, emitted_at=now_millis()) + return AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 records_in_sync = 0 - records_in_slice = 0 median_record_byte_size = 450 yield generate_estimate(self.name, self.count - total_records, median_record_byte_size) - for i in range(total_records, self.count): - user = self.generate_user(i) - yield user - total_records += 1 - records_in_sync += 1 - records_in_slice += 1 + running = True + with Pool(initializer=self.worker_init, processes=self.threads) as pool: + while running and records_in_sync < self.count: + records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) + if records_remaining_this_loop == 0: + break + users = pool.map(self.generate_user, range(total_records, total_records + records_remaining_this_loop)) + for user in users: + total_records += 1 + records_in_sync += 1 + yield user + + if records_in_sync == self.records_per_sync: + running = False + break - if records_in_slice >= self.records_per_slice: self.state = {self.cursor_field: total_records, "seed": self.seed} - records_in_slice = 0 - if records_in_sync == self.records_per_sync: - break - self.state = {self.cursor_field: total_records, "seed": self.seed} - - -class Purchases(Stream, IncrementalMixin): +class Purchases(Stream, IncrementalMixin, FakerMultithreaded): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, threads:int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed @@ -156,6 +174,7 @@ def __init__(self, count: int, seed: int, records_per_sync: int, records_per_sli self.records_per_slice = records_per_slice self.dt = Datetime(seed=self.seed) self.numeric = Numeric(seed=self.seed) + self.threads = threads @property def state_checkpoint_interval(self) -> Optional[int]: @@ -183,72 +202,100 @@ def random_date_in_range( random_date = start_date + datetime.timedelta(days=random_number_of_days) return random_date - def generate_purchases(self, user_id: int, purchases_count: int) -> list[Dict]: + def generate_purchases(self, user_id: int) -> list[Dict]: + """Because we are doing this work in parallel processes, we need a deterministic way to know what a purchase's ID should be given on the input of a user_id""" + """tldr; Every 10 user_ids produce 10 purchases. User ID x5 has no purchases, User ID mod x7 has 2, and everyone else has 1""" + purchases: list[Dict] = [] - purchase_percent_remaining = 70 # ~30% of people will have no purchases + last_user_id_digit = int(repr(user_id)[-1]) + purchase_count = 1 + id_offset = 0 + if last_user_id_digit - 1 == 5: + purchase_count = 0 + elif last_user_id_digit - 1 == 6: + id_offset = 1 + elif last_user_id_digit - 1 == 7: + id_offset = 1 + purchase_count = 2 + total_products = 100 - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) i = 0 - time_a = self.dt.datetime() - time_b = self.dt.datetime() - created_at = time_a if time_a <= time_b else time_b - - while purchase_percent_remaining > 0: - id = purchases_count + i + 1 - product_id = self.numeric.integer_number(1, total_products) + while purchase_count > 0: + id = user_id + i + 1 - id_offset + time_a = dt.datetime() + time_b = dt.datetime() + created_at = time_a if time_a <= time_b else time_b + product_id = numeric.integer_number(1, total_products) added_to_cart_at = self.random_date_in_range(created_at) purchased_at = ( self.random_date_in_range(added_to_cart_at) - if added_to_cart_at is not None and self.numeric.integer_number(1, 100) <= 70 + if added_to_cart_at is not None and numeric.integer_number(1, 100) <= 70 else None ) # 70% likely to purchase the item in the cart returned_at = ( - self.random_date_in_range(purchased_at) if purchased_at is not None and self.numeric.integer_number(1, 100) <= 15 else None + self.random_date_in_range(purchased_at) if purchased_at is not None and numeric.integer_number(1, 100) <= 15 else None ) # 15% likely to return the item purchase = { "id": id, "product_id": product_id, - "user_id": user_id, + "user_id": user_id + 1, "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, } - purchases.append(purchase) - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) + record = AirbyteRecordMessage(stream=self.name, data=purchase, emitted_at=now_millis()) + message = AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) + purchases.append(message) + + purchase_count = purchase_count - 1 i += 1 + return purchases def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 total_user_records = self.state["user_id"] if "user_id" in self.state else 0 user_records_in_sync = 0 - user_records_in_slice = 0 median_record_byte_size = 230 yield generate_estimate( self.name, (self.count - total_user_records) * 1.3, median_record_byte_size ) # a fuzzy guess, some users have purchases, some don't - for i in range(total_user_records, self.count): - purchases = self.generate_purchases(i + 1, total_purchase_records) - for purchase in purchases: - total_purchase_records += 1 - yield purchase - total_user_records += 1 - user_records_in_sync += 1 - user_records_in_slice += 1 + running = True + with Pool(initializer=self.worker_init, processes=self.threads) as pool: + while running and total_user_records < self.count: + records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) + if records_remaining_this_loop == 0: + break + carts = pool.map(self.generate_purchases, range(total_user_records, total_user_records + records_remaining_this_loop)) + for purchases in carts: + for purchase in purchases: + yield purchase + + total_user_records += 1 + user_records_in_sync += 1 + + if user_records_in_sync == self.records_per_sync: + running = False + break - if user_records_in_slice >= self.records_per_slice: self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} - user_records_in_slice = 0 - if user_records_in_sync == self.records_per_sync: - break +def now_millis(): + return int(datetime.datetime.now().timestamp() * 1000) + +class AirbyteMessageWithCachedJSON(AirbyteMessage): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._json = self.json(exclude_unset=True) + self.json = self.get_json - self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + def get_json(self, **kwargs): + return self._json def generate_estimate(stream_name: str, total: int, bytes_per_row: int): diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index 8a989b7778ac..34b391dfae79 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -25,7 +25,7 @@ def exception(a,b,**kwargs): def schemas_are_valid(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "threads": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -36,7 +36,7 @@ def schemas_are_valid(): def test_source_streams(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "threads": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -64,7 +64,7 @@ def test_source_streams(): def test_read_small_random_data(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "threads": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -98,7 +98,7 @@ def test_read_small_random_data(): def test_no_read_limit_hit(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "threads": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -123,12 +123,12 @@ def test_no_read_limit_hit(): assert record_rows_count == 0 assert state_rows_count == 1 - assert latest_state.state.data == {"users": {"id": 10, "seed": None}} + assert latest_state.state.data == {"users": {"id": 10}} def test_read_big_random_data(): source = SourceFaker() - config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000} + config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000, "threads": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -163,7 +163,7 @@ def test_read_big_random_data(): def test_with_purchases(): source = SourceFaker() - config = {"count": 1000, "records_per_sync": 1000} + config = {"count": 1000, "records_per_sync": 1000, "threads": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -205,7 +205,7 @@ def test_with_purchases(): def test_sync_ends_with_limit(): source = SourceFaker() - config = {"count": 100, "records_per_sync": 5} + config = {"count": 100, "records_per_sync": 5, "threads": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -239,7 +239,7 @@ def test_read_with_seed(): """ source = SourceFaker() - config = {"count": 1, "seed": 100} + config = {"count": 1, "seed": 100, "threads": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -253,14 +253,14 @@ def test_read_with_seed(): iterator = source.read(logger, config, catalog, state) records = [row for row in iterator if row.type is Type.RECORD] - assert records[0].record.data["occupation"] == "Roadworker" - assert records[0].record.data["email"] == "reproduce1856@outlook.com" + assert records[0].record.data["occupation"] == "Illustrator" + assert records[0].record.data["email"] == "upcoming1928@protonmail.com" def test_ensure_no_purchases_without_users(): with pytest.raises(ValueError): source = SourceFaker() - config = {"count": 100} + config = {"count": 100, "threads": 1} catalog = ConfiguredAirbyteCatalog( streams=[ {"stream": {"name": "purchases", "json_schema": {}}, "sync_mode": "incremental", "destination_sync_mode": "overwrite"}, From 77de03f1a6bded0de38316a2bd5ecdc22fbf58b4 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 20 Dec 2022 16:12:30 -0800 Subject: [PATCH 02/13] threads in spec + lint --- .../source-faker/source_faker/spec.json | 8 ++++++ .../source-faker/source_faker/streams.py | 28 ++++++++++++------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index 1018a957f109..c9fc7c8267d0 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -37,6 +37,14 @@ "minimum": 1, "default": 100, "order": 3 + }, + "threads": { + "title": "Worker Threads", + "description": "How many parallel worker threads should be used to generate records?", + "type": "integer", + "minimum": 1, + "default": 4, + "order": 4 } } } diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 187f50c35b91..739180a44a51 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -4,19 +4,25 @@ import datetime import os - from multiprocessing import Pool, current_process -# from multiprocessing.pool import ThreadPool - -from typing import Any, Dict, Iterable, Mapping, Optional, List - -from airbyte_cdk.models import AirbyteMessage, AirbyteEstimateTraceMessage, Type, AirbyteTraceMessage, AirbyteRecordMessage, EstimateType, TraceType +from typing import Any, Dict, Iterable, Mapping, Optional + +from airbyte_cdk.models import ( + AirbyteEstimateTraceMessage, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteTraceMessage, + EstimateType, + TraceType, + Type, +) from airbyte_cdk.sources.streams import IncrementalMixin, Stream from mimesis import Datetime, Numeric, Person from mimesis.locales import Locale from .utils import format_airbyte_time, read_json + class FakerMultithreaded: def worker_init(self): """For the workers, we want a unique instance of the faker packages with their own seeds to differentiate the generated responses""" @@ -24,7 +30,7 @@ def worker_init(self): global dt global numeric seed_with_offset = self.seed - if self.seed != None: + if self.seed is not None: seed_with_offset = self.seed + current_process()._identity[0] person = Person(locale=Locale.EN, seed=seed_with_offset) dt = Datetime(seed=seed_with_offset) @@ -35,7 +41,7 @@ class Products(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, threads:int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.seed = seed self.records_per_sync = records_per_sync @@ -81,7 +87,7 @@ class Users(Stream, IncrementalMixin, FakerMultithreaded): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, threads:int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed @@ -166,7 +172,7 @@ class Purchases(Stream, IncrementalMixin, FakerMultithreaded): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, threads:int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed @@ -285,9 +291,11 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + def now_millis(): return int(datetime.datetime.now().timestamp() * 1000) + class AirbyteMessageWithCachedJSON(AirbyteMessage): def __init__(self, **kwargs): super().__init__(**kwargs) From e1200afa962c8cf5f4d1af568d250c3e3ce67a4c Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 20 Dec 2022 16:28:56 -0800 Subject: [PATCH 03/13] pass tests --- .../connectors/source-faker/source_faker/streams.py | 11 +++++++++-- .../connectors/source-faker/unit_tests/unit_test.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 739180a44a51..21ecc31ff488 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -153,7 +153,8 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: with Pool(initializer=self.worker_init, processes=self.threads) as pool: while running and records_in_sync < self.count: records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) - if records_remaining_this_loop == 0: + if records_remaining_this_loop <= 0: + running = False break users = pool.map(self.generate_user, range(total_records, total_records + records_remaining_this_loop)) for user in users: @@ -167,6 +168,8 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.state = {self.cursor_field: total_records, "seed": self.seed} + self.state = {self.cursor_field: total_records, "seed": self.seed} + class Purchases(Stream, IncrementalMixin, FakerMultithreaded): primary_key = None @@ -275,11 +278,13 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: with Pool(initializer=self.worker_init, processes=self.threads) as pool: while running and total_user_records < self.count: records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) - if records_remaining_this_loop == 0: + if records_remaining_this_loop <= 0: + running = False break carts = pool.map(self.generate_purchases, range(total_user_records, total_user_records + records_remaining_this_loop)) for purchases in carts: for purchase in purchases: + total_purchase_records += 1 yield purchase total_user_records += 1 @@ -291,6 +296,8 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + def now_millis(): return int(datetime.datetime.now().timestamp() * 1000) diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index 34b391dfae79..2a21bd69530a 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -123,7 +123,7 @@ def test_no_read_limit_hit(): assert record_rows_count == 0 assert state_rows_count == 1 - assert latest_state.state.data == {"users": {"id": 10}} + assert latest_state.state.data == {"users": {"id": 10, "seed": None}} def test_read_big_random_data(): From 1660099fa0015a509a7d6c01eeec18d5a94f2264 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 21 Dec 2022 15:03:23 -0800 Subject: [PATCH 04/13] revert changes to record helper --- airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py index 71693ca73a13..482596230f04 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py @@ -20,9 +20,7 @@ def stream_data_to_airbyte_message( if schema is None: schema = {} - if isinstance(data_or_message, AirbyteMessage): - return data_or_message - elif isinstance(data_or_message, dict): + if isinstance(data_or_message, dict): data = data_or_message now_millis = int(datetime.datetime.now().timestamp() * 1000) # Transform object fields according to config. Most likely you will From 104d7dd1c33174cf7290948a3c8907d655b3bc97 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 21 Dec 2022 15:14:53 -0800 Subject: [PATCH 05/13] cleanup --- .../airbyte_message_with_cached_json.py | 17 +++++++++ .../source-faker/source_faker/streams.py | 35 ++----------------- .../source-faker/source_faker/utils.py | 14 ++++++++ 3 files changed, 34 insertions(+), 32 deletions(-) create mode 100644 airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py diff --git a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py new file mode 100644 index 000000000000..5567599f14de --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.models import AirbyteMessage + + +class AirbyteMessageWithCachedJSON(AirbyteMessage): + """I a monkeypatch to AirbyteMessage which pre-renders the JSON-representation of the object upon initialization. This allows the JSON to be calculated in the thread/process that builds the object rather than the main thread.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._json = self.json(exclude_unset=True) + self.json = self.get_json + + def get_json(self, **kwargs): + return self._json diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 21ecc31ff488..960c80f9c02b 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -7,20 +7,13 @@ from multiprocessing import Pool, current_process from typing import Any, Dict, Iterable, Mapping, Optional -from airbyte_cdk.models import ( - AirbyteEstimateTraceMessage, - AirbyteMessage, - AirbyteRecordMessage, - AirbyteTraceMessage, - EstimateType, - TraceType, - Type, -) +from airbyte_cdk.models import AirbyteRecordMessage, Type from airbyte_cdk.sources.streams import IncrementalMixin, Stream from mimesis import Datetime, Numeric, Person from mimesis.locales import Locale -from .utils import format_airbyte_time, read_json +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, generate_estimate, now_millis, read_json class FakerMultithreaded: @@ -297,25 +290,3 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} - - -def now_millis(): - return int(datetime.datetime.now().timestamp() * 1000) - - -class AirbyteMessageWithCachedJSON(AirbyteMessage): - def __init__(self, **kwargs): - super().__init__(**kwargs) - self._json = self.json(exclude_unset=True) - self.json = self.get_json - - def get_json(self, **kwargs): - return self._json - - -def generate_estimate(stream_name: str, total: int, bytes_per_row: int): - emitted_at = int(datetime.datetime.now().timestamp() * 1000) - estimate_message = AirbyteEstimateTraceMessage( - type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) - ) - return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/utils.py b/airbyte-integrations/connectors/source-faker/source_faker/utils.py index 12970853c956..9c094b11bcb7 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/utils.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/utils.py @@ -5,6 +5,8 @@ import datetime import json +from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType + def read_json(filepath): with open(filepath, "r") as f: @@ -17,3 +19,15 @@ def format_airbyte_time(d: datetime): s = s.replace(" ", "T") s += "+00:00" return s + + +def now_millis(): + return int(datetime.datetime.now().timestamp() * 1000) + + +def generate_estimate(stream_name: str, total: int, bytes_per_row: int): + emitted_at = int(datetime.datetime.now().timestamp() * 1000) + estimate_message = AirbyteEstimateTraceMessage( + type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) + ) + return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) From 5becbc1010c7ba115eef36faa5d50abab972a36d Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 21 Dec 2022 15:37:06 -0800 Subject: [PATCH 06/13] update expected_records --- .../integration_tests/expected_records.jsonl | 243 +++++++++--------- 1 file changed, 120 insertions(+), 123 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index 87ff4c896424..193e54ef3713 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -1,123 +1,120 @@ -{"stream": "users", "data": {"id": 1, "created_at": "2009-08-12T18:57:58+00:00", "updated_at": "2012-07-02T08:32:31+00:00", "name": "Reda", "title": "M.Sc.Tech.", "age": 47, "email": "locations1983@protonmail.com", "telephone": "+1-(110)-795-7610", "gender": "Male", "language": "Tamil", "academic_degree": "Master", "nationality": "Italian", "occupation": "Word Processing Operator", "height": "1.55", "blood_type": "B\u2212", "weight": 58}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 2, "created_at": "2008-09-23T19:57:09+00:00", "updated_at": "2016-03-10T04:48:06+00:00", "name": "Tristan", "title": "M.Sc.Tech.", "age": 32, "email": "variations1847@duck.com", "telephone": "683-770-9281", "gender": "Other", "language": "Bosnian", "academic_degree": "Bachelor", "nationality": "Estonian", "occupation": "Tiler", "height": "2.00", "blood_type": "AB\u2212", "weight": 44}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 3, "created_at": "2003-06-14T10:39:40+00:00", "updated_at": "2003-12-03T21:21:30+00:00", "name": "Yuki", "title": "Miss", "age": 50, "email": "vacuum2027@yahoo.com", "telephone": "1-321-809-2061", "gender": "Female", "language": "Armenian", "academic_degree": "Bachelor", "nationality": "Swiss", "occupation": "Valuer", "height": "1.84", "blood_type": "O\u2212", "weight": 71}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 4, "created_at": "2001-09-30T00:05:46+00:00", "updated_at": "2006-09-16T14:55:33+00:00", "name": "Fred", "title": "MMath", "age": 47, "email": "causes1859@outlook.com", "telephone": "(827) 127-3811", "gender": "Female", "language": "Assamese", "academic_degree": "PhD", "nationality": "Russian", "occupation": "Turkey Farmer", "height": "1.80", "blood_type": "A+", "weight": 39}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 5, "created_at": "2012-12-27T21:40:00+00:00", "updated_at": "2015-06-08T23:20:45+00:00", "name": "Emmitt", "title": "DPhil", "age": 39, "email": "athens1899@gmail.com", "telephone": "(470) 656-8003", "gender": "Other", "language": "English", "academic_degree": "Bachelor", "nationality": "Jordanian", "occupation": "Stone Sawyer", "height": "1.52", "blood_type": "A+", "weight": 82}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 6, "created_at": "2002-04-30T18:14:15+00:00", "updated_at": "2004-09-15T02:05:20+00:00", "name": "Hollis", "title": "MSc", "age": 52, "email": "fisheries1881@yandex.com", "telephone": "(519) 606-9896", "gender": "Other", "language": "Swati", "academic_degree": "Master", "nationality": "Chilean", "occupation": "Writer", "height": "1.85", "blood_type": "AB\u2212", "weight": 85}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 7, "created_at": "2003-09-11T17:13:51+00:00", "updated_at": "2016-08-04T09:35:18+00:00", "name": "Kip", "title": "M.D.", "age": 31, "email": "numbers1983@example.com", "telephone": "346-013-2638", "gender": "Other", "language": "Armenian", "academic_degree": "Master", "nationality": "Swiss", "occupation": "Salesman", "height": "1.89", "blood_type": "O+", "weight": 48}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 8, "created_at": "2012-06-19T07:18:11+00:00", "updated_at": "2017-10-10T14:05:38+00:00", "name": "Carie", "title": "Madam", "age": 49, "email": "watershed1819@example.com", "telephone": "(348) 881-9607", "gender": "Male", "language": "Kyrgyz", "academic_degree": "PhD", "nationality": "Guatemalan", "occupation": "Park Ranger", "height": "1.50", "blood_type": "O+", "weight": 83}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 9, "created_at": "2002-11-25T04:56:09+00:00", "updated_at": "2005-01-20T21:16:30+00:00", "name": "Steven", "title": "Mr.", "age": 54, "email": "llp1893@yahoo.com", "telephone": "830.247.8156", "gender": "Fluid", "language": "Catalan", "academic_degree": "Bachelor", "nationality": "Egyptian", "occupation": "Ambulance Driver", "height": "1.52", "blood_type": "AB+", "weight": 81}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 10, "created_at": "2001-02-23T17:43:25+00:00", "updated_at": "2022-09-09T16:51:15+00:00", "name": "Lore", "title": "Madam", "age": 61, "email": "resident2075@example.com", "telephone": "321.233.0702", "gender": "Female", "language": "Polish", "academic_degree": "Master", "nationality": "French", "occupation": "Registrar", "height": "1.99", "blood_type": "B+", "weight": 56}, "emitted_at": 1669830193008} -{"stream": "purchases", "data": {"id": 1, "product_id": 98, "user_id": 1, "added_to_cart_at": "2019-01-17T18:57:58+00:00", "purchased_at": "2020-06-30T18:57:58+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 2, "product_id": 39, "user_id": 2, "added_to_cart_at": "2019-06-02T19:57:09+00:00", "purchased_at": "2022-09-08T19:57:09+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 3, "product_id": 37, "user_id": 3, "added_to_cart_at": "2006-08-01T10:39:40+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 4, "product_id": 80, "user_id": 3, "added_to_cart_at": "2021-05-18T10:39:40+00:00", "purchased_at": "2022-11-14T10:39:40+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 5, "product_id": 40, "user_id": 4, "added_to_cart_at": "2003-12-18T00:05:46+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 6, "product_id": 88, "user_id": 4, "added_to_cart_at": "2009-02-24T00:05:46+00:00", "purchased_at": "2021-09-14T00:05:46+00:00", "returned_at": "2022-03-14T00:05:46+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 7, "product_id": 79, "user_id": 5, "added_to_cart_at": "2020-03-03T21:40:00+00:00", "purchased_at": "2022-11-17T21:40:00+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 8, "product_id": 67, "user_id": 6, "added_to_cart_at": "2008-03-02T18:14:15+00:00", "purchased_at": "2020-06-21T18:14:15+00:00", "returned_at": "2020-09-24T18:14:15+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 9, "product_id": 91, "user_id": 7, "added_to_cart_at": "2022-03-12T17:13:51+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 10, "product_id": 79, "user_id": 8, "added_to_cart_at": "2017-12-31T07:18:11+00:00", "purchased_at": "2019-05-14T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 11, "product_id": 91, "user_id": 8, "added_to_cart_at": "2022-03-24T07:18:11+00:00", "purchased_at": "2022-06-29T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 12, "product_id": 19, "user_id": 9, "added_to_cart_at": "2020-11-29T04:56:09+00:00", "purchased_at": "2022-03-02T04:56:09+00:00", "returned_at": "2022-04-12T04:56:09+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 13, "product_id": 63, "user_id": 10, "added_to_cart_at": "2003-08-05T17:43:25+00:00", "purchased_at": "2015-12-15T17:43:25+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1669830193015} +{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Walter", "title": "PhD", "age": 47, "email": "similarly1907@example.com", "telephone": "084.704.7299", "gender": "Male", "language": "Amharic", "academic_degree": "Bachelor", "nationality": "Saudi", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1671665757124} +{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Nestor", "title": "Sir", "age": 47, "email": "guys1918@example.org", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Arnita", "title": "Madam", "age": 42, "email": "painted1991@yandex.com", "telephone": "611.837.4386", "gender": "Other", "language": "Spanish", "academic_degree": "Bachelor", "nationality": "Chinese", "occupation": "Textile Engineer", "height": "1.61", "blood_type": "O+", "weight": 87}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 4, "created_at": "2000-09-08T14:31:35+00:00", "updated_at": "2011-04-22T07:48:29+00:00", "name": "Theron", "title": "M.Sc.Tech.", "age": 38, "email": "sectors2080@outlook.com", "telephone": "1-788-785-5408", "gender": "Female", "language": "Montenegrin", "academic_degree": "Master", "nationality": "Japanese", "occupation": "Printer", "height": "1.71", "blood_type": "O+", "weight": 72}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 5, "created_at": "2005-11-24T09:07:47+00:00", "updated_at": "2009-01-14T17:59:41+00:00", "name": "Sharika", "title": "Master", "age": 27, "email": "espn2082@example.org", "telephone": "860-007-2163", "gender": "Male", "language": "Dutch", "academic_degree": "Bachelor", "nationality": "Ecuadorian", "occupation": "Travel Courier", "height": "1.98", "blood_type": "O\u2212", "weight": 79}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 6, "created_at": "2021-04-10T09:37:56+00:00", "updated_at": "2022-09-30T13:32:53+00:00", "name": "Shanita", "title": "MSc", "age": 23, "email": "mali1975@duck.com", "telephone": "227.994.9230", "gender": "Female", "language": "Malagasy", "academic_degree": "PhD", "nationality": "Jordanian", "occupation": "Security Consultant", "height": "1.77", "blood_type": "AB+", "weight": 78}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 7, "created_at": "2012-07-22T05:23:35+00:00", "updated_at": "2016-07-19T01:30:15+00:00", "name": "Jen", "title": "LL.D", "age": 58, "email": "blogs1864@protonmail.com", "telephone": "383-974-2080", "gender": "Female", "language": "Pashto", "academic_degree": "Master", "nationality": "Chinese", "occupation": "Timber Worker", "height": "1.52", "blood_type": "AB+", "weight": 60}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 8, "created_at": "2016-02-25T05:33:53+00:00", "updated_at": "2022-11-24T11:05:28+00:00", "name": "Fermin", "title": "Madam", "age": 22, "email": "letting2055@protonmail.com", "telephone": "(642) 188-5142", "gender": "Male", "language": "Khmer", "academic_degree": "PhD", "nationality": "Finnish", "occupation": "Turkey Farmer", "height": "1.74", "blood_type": "AB+", "weight": 42}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 9, "created_at": "2011-08-24T00:30:02+00:00", "updated_at": "2022-10-19T18:25:41+00:00", "name": "Cecil", "title": "Mr.", "age": 50, "email": "interference2059@yahoo.com", "telephone": "(312) 977-4573", "gender": "Other", "language": "Bengali", "academic_degree": "Master", "nationality": "Chilean", "occupation": "Foreman", "height": "1.67", "blood_type": "B\u2212", "weight": 42}, "emitted_at": 1671665757125} +{"stream": "users", "data": {"id": 10, "created_at": "2005-09-08T00:49:12+00:00", "updated_at": "2017-04-13T16:22:54+00:00", "name": "Von", "title": "M.A.", "age": 34, "email": "consequently1941@example.org", "telephone": "806-018-0294", "gender": "Male", "language": "Malagasy", "academic_degree": "Bachelor", "nationality": "Spanish", "occupation": "Machinist", "height": "1.58", "blood_type": "A+", "weight": 65}, "emitted_at": 1671665757125} +{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2020-01-12T11:53:10+00:00", "purchased_at": "2022-03-10T11:53:10+00:00", "returned_at": "2022-10-11T11:53:10+00:00"}, "emitted_at": 1671665757354} +{"stream": "purchases", "data": {"id": 2, "product_id": 47, "user_id": 2, "added_to_cart_at": "2016-11-30T14:32:17+00:00", "purchased_at": "2019-10-26T14:32:17+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 3, "product_id": 86, "user_id": 3, "added_to_cart_at": "2022-09-08T14:59:20+00:00", "purchased_at": "2022-11-12T14:59:20+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 4, "product_id": 78, "user_id": 4, "added_to_cart_at": "2015-11-20T05:20:11+00:00", "purchased_at": "2020-08-07T05:20:11+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 5, "product_id": 5, "user_id": 5, "added_to_cart_at": "2012-03-25T11:54:28+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 6, "product_id": 88, "user_id": 6, "added_to_cart_at": "2015-11-19T14:41:33+00:00", "purchased_at": "2021-08-13T14:41:33+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 7, "product_id": 82, "user_id": 8, "added_to_cart_at": "2021-04-12T17:46:29+00:00", "purchased_at": "2022-05-11T17:46:29+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 8, "product_id": 66, "user_id": 9, "added_to_cart_at": "2021-10-02T15:19:19+00:00", "purchased_at": "2021-10-22T15:19:19+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 9, "product_id": 57, "user_id": 9, "added_to_cart_at": "2019-12-22T23:13:31+00:00", "purchased_at": "2022-11-03T23:13:31+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "purchases", "data": {"id": 10, "product_id": 5, "user_id": 10, "added_to_cart_at": "2017-08-04T03:03:36+00:00", "purchased_at": "2021-11-03T03:03:36+00:00", "returned_at": null}, "emitted_at": 1671665757355} +{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1671665757360} +{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1671665757361} +{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1671665757362} +{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1671665757363} +{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1671665757363} From 4c60b0770c336b7597022368d7d85f8c137a008f Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 21 Dec 2022 15:51:50 -0800 Subject: [PATCH 07/13] bump default records-per-slice to 1k --- .../connectors/source-faker/source_faker/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index c9fc7c8267d0..0715a4ac366c 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -35,7 +35,7 @@ "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", "type": "integer", "minimum": 1, - "default": 100, + "default": 1000, "order": 3 }, "threads": { From f2f8e6f19950ed9c8079e13fb584e7a9eccaa85c Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 21 Dec 2022 16:05:31 -0800 Subject: [PATCH 08/13] enforce unique email addresses --- .../connectors/source-faker/source_faker/streams.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 960c80f9c02b..a09d48461bd9 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -107,6 +107,10 @@ def generate_user(self, user_id: int): time_a = dt.datetime() time_b = dt.datetime() + # faker doesn't always produce unique email addresses, so to enforce uniqueness, we will append the user_id to the prefix + email_parts = person.email().split("@") + email = f"{email_parts[0]}+{user_id + 1}@{email_parts[1]}" + profile = { "id": user_id + 1, "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), @@ -114,7 +118,7 @@ def generate_user(self, user_id: int): "name": person.name(), "title": person.title(), "age": person.age(), - "email": person.email(), + "email": email, "telephone": person.telephone(), "gender": person.gender(), "language": person.language(), From da4c02e17458fe750fadaa74bb27f7daec9d53bf Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 22 Dec 2022 13:43:08 -0800 Subject: [PATCH 09/13] cleanup --- .../airbyte_message_with_cached_json.py | 7 +- .../source_faker/purchase_generator.py | 103 ++++++++++++ .../source-faker/source_faker/streams.py | 156 ++---------------- .../source_faker/user_generator.py | 72 ++++++++ 4 files changed, 197 insertions(+), 141 deletions(-) create mode 100644 airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py create mode 100644 airbyte-integrations/connectors/source-faker/source_faker/user_generator.py diff --git a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py index 5567599f14de..316cbe76405f 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py @@ -6,7 +6,12 @@ class AirbyteMessageWithCachedJSON(AirbyteMessage): - """I a monkeypatch to AirbyteMessage which pre-renders the JSON-representation of the object upon initialization. This allows the JSON to be calculated in the thread/process that builds the object rather than the main thread.""" + """ + I a monkeypatch to AirbyteMessage which pre-renders the JSON-representation of the object upon initialization. + This allows the JSON to be calculated in the thread/process that builds the object rather than the main thread. + + Note: We can't use @cache here because the LRU cache is not serializable when passed to child workers. + """ def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py new file mode 100644 index 000000000000..b1a206a33bf8 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from multiprocessing import current_process +from typing import Dict + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Numeric + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class PurchaseGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same thread, resetting the internal random counters + """ + + seed_with_offset = self.seed + if len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global dt + global numeric + + dt = Datetime(seed=seed_with_offset) + numeric = Numeric(seed=seed_with_offset) + + def random_date_in_range( + self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() + ) -> datetime.datetime: + time_between_dates = end_date - start_date + days_between_dates = time_between_dates.days + if days_between_dates < 2: + days_between_dates = 2 + random_number_of_days = numeric.integer_number(0, days_between_dates) + random_date = start_date + datetime.timedelta(days=random_number_of_days) + return random_date + + def generate(self, user_id: int) -> list[Dict]: + """ + Because we are doing this work in parallel processes, we need a deterministic way to know what a purchase's ID should be given on the input of a user_id. + tldr; Every 10 user_ids produce 10 purchases. User ID x5 has no purchases, User ID mod x7 has 2, and everyone else has 1 + """ + + purchases: list[Dict] = [] + last_user_id_digit = int(repr(user_id)[-1]) + purchase_count = 1 + id_offset = 0 + if last_user_id_digit - 1 == 5: + purchase_count = 0 + elif last_user_id_digit - 1 == 6: + id_offset = 1 + elif last_user_id_digit - 1 == 7: + id_offset = 1 + purchase_count = 2 + + total_products = 100 + i = 0 + + while purchase_count > 0: + id = user_id + i + 1 - id_offset + time_a = dt.datetime() + time_b = dt.datetime() + created_at = time_a if time_a <= time_b else time_b + product_id = numeric.integer_number(1, total_products) + added_to_cart_at = self.random_date_in_range(created_at) + purchased_at = ( + self.random_date_in_range(added_to_cart_at) + if added_to_cart_at is not None and numeric.integer_number(1, 100) <= 70 + else None + ) # 70% likely to purchase the item in the cart + returned_at = ( + self.random_date_in_range(purchased_at) if purchased_at is not None and numeric.integer_number(1, 100) <= 15 else None + ) # 15% likely to return the item + + purchase = { + "id": id, + "product_id": product_id, + "user_id": user_id + 1, + "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, + "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, + "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, + } + + record = AirbyteRecordMessage(stream=self.stream_name, data=purchase, emitted_at=now_millis()) + message = AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) + purchases.append(message) + + purchase_count = purchase_count - 1 + i += 1 + + return purchases diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index a09d48461bd9..fcc7b9a9f9dc 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -2,32 +2,15 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import datetime import os -from multiprocessing import Pool, current_process +from multiprocessing import Pool from typing import Any, Dict, Iterable, Mapping, Optional -from airbyte_cdk.models import AirbyteRecordMessage, Type from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from mimesis import Datetime, Numeric, Person -from mimesis.locales import Locale -from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON -from .utils import format_airbyte_time, generate_estimate, now_millis, read_json - - -class FakerMultithreaded: - def worker_init(self): - """For the workers, we want a unique instance of the faker packages with their own seeds to differentiate the generated responses""" - global person - global dt - global numeric - seed_with_offset = self.seed - if self.seed is not None: - seed_with_offset = self.seed + current_process()._identity[0] - person = Person(locale=Locale.EN, seed=seed_with_offset) - dt = Datetime(seed=seed_with_offset) - numeric = Numeric(seed=seed_with_offset) +from .purchase_generator import PurchaseGenerator +from .user_generator import UserGenerator +from .utils import generate_estimate, read_json class Products(Stream, IncrementalMixin): @@ -76,7 +59,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.state = {self.cursor_field: total_records, "seed": self.seed} -class Users(Stream, IncrementalMixin, FakerMultithreaded): +class Users(Stream, IncrementalMixin): primary_key = None cursor_field = "id" @@ -87,6 +70,7 @@ def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, r self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice self.threads = threads + self.generator = UserGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -103,42 +87,6 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def generate_user(self, user_id: int): - time_a = dt.datetime() - time_b = dt.datetime() - - # faker doesn't always produce unique email addresses, so to enforce uniqueness, we will append the user_id to the prefix - email_parts = person.email().split("@") - email = f"{email_parts[0]}+{user_id + 1}@{email_parts[1]}" - - profile = { - "id": user_id + 1, - "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), - "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), - "name": person.name(), - "title": person.title(), - "age": person.age(), - "email": email, - "telephone": person.telephone(), - "gender": person.gender(), - "language": person.language(), - "academic_degree": person.academic_degree(), - "nationality": person.nationality(), - "occupation": person.occupation(), - "height": person.height(), - "blood_type": person.blood_type(), - "weight": person.weight(), - } - - while not profile["created_at"]: - profile["created_at"] = format_airbyte_time(dt.datetime()) - - if not profile["updated_at"]: - profile["updated_at"] = profile["created_at"] + 1 - - record = AirbyteRecordMessage(stream=self.name, data=profile, emitted_at=now_millis()) - return AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 records_in_sync = 0 @@ -146,21 +94,18 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: median_record_byte_size = 450 yield generate_estimate(self.name, self.count - total_records, median_record_byte_size) - running = True - with Pool(initializer=self.worker_init, processes=self.threads) as pool: - while running and records_in_sync < self.count: + with Pool(initializer=self.generator.prepare, processes=self.threads) as pool: + while records_in_sync < self.count: records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) if records_remaining_this_loop <= 0: - running = False break - users = pool.map(self.generate_user, range(total_records, total_records + records_remaining_this_loop)) + users = pool.map(self.generator.generate, range(total_records, total_records + records_remaining_this_loop)) for user in users: total_records += 1 records_in_sync += 1 yield user if records_in_sync == self.records_per_sync: - running = False break self.state = {self.cursor_field: total_records, "seed": self.seed} @@ -168,7 +113,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.state = {self.cursor_field: total_records, "seed": self.seed} -class Purchases(Stream, IncrementalMixin, FakerMultithreaded): +class Purchases(Stream, IncrementalMixin): primary_key = None cursor_field = "id" @@ -178,9 +123,8 @@ def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, r self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.dt = Datetime(seed=self.seed) - self.numeric = Numeric(seed=self.seed) self.threads = threads + self.generator = PurchaseGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -197,88 +141,21 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def random_date_in_range( - self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() - ) -> datetime.datetime: - time_between_dates = end_date - start_date - days_between_dates = time_between_dates.days - if days_between_dates < 2: - days_between_dates = 2 - random_number_of_days = self.numeric.integer_number(0, days_between_dates) - random_date = start_date + datetime.timedelta(days=random_number_of_days) - return random_date - - def generate_purchases(self, user_id: int) -> list[Dict]: - """Because we are doing this work in parallel processes, we need a deterministic way to know what a purchase's ID should be given on the input of a user_id""" - """tldr; Every 10 user_ids produce 10 purchases. User ID x5 has no purchases, User ID mod x7 has 2, and everyone else has 1""" - - purchases: list[Dict] = [] - last_user_id_digit = int(repr(user_id)[-1]) - purchase_count = 1 - id_offset = 0 - if last_user_id_digit - 1 == 5: - purchase_count = 0 - elif last_user_id_digit - 1 == 6: - id_offset = 1 - elif last_user_id_digit - 1 == 7: - id_offset = 1 - purchase_count = 2 - - total_products = 100 - i = 0 - - while purchase_count > 0: - id = user_id + i + 1 - id_offset - time_a = dt.datetime() - time_b = dt.datetime() - created_at = time_a if time_a <= time_b else time_b - product_id = numeric.integer_number(1, total_products) - added_to_cart_at = self.random_date_in_range(created_at) - purchased_at = ( - self.random_date_in_range(added_to_cart_at) - if added_to_cart_at is not None and numeric.integer_number(1, 100) <= 70 - else None - ) # 70% likely to purchase the item in the cart - returned_at = ( - self.random_date_in_range(purchased_at) if purchased_at is not None and numeric.integer_number(1, 100) <= 15 else None - ) # 15% likely to return the item - - purchase = { - "id": id, - "product_id": product_id, - "user_id": user_id + 1, - "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, - "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, - "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, - } - - record = AirbyteRecordMessage(stream=self.name, data=purchase, emitted_at=now_millis()) - message = AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) - purchases.append(message) - - purchase_count = purchase_count - 1 - i += 1 - - return purchases - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 total_user_records = self.state["user_id"] if "user_id" in self.state else 0 user_records_in_sync = 0 + # a fuzzy guess, some users have purchases, some don't median_record_byte_size = 230 - yield generate_estimate( - self.name, (self.count - total_user_records) * 1.3, median_record_byte_size - ) # a fuzzy guess, some users have purchases, some don't + yield generate_estimate(self.name, (self.count - total_user_records) * 1.3, median_record_byte_size) - running = True - with Pool(initializer=self.worker_init, processes=self.threads) as pool: - while running and total_user_records < self.count: + with Pool(initializer=self.generator.prepare, processes=self.threads) as pool: + while total_user_records < self.count: records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) if records_remaining_this_loop <= 0: - running = False break - carts = pool.map(self.generate_purchases, range(total_user_records, total_user_records + records_remaining_this_loop)) + carts = pool.map(self.generator.generate, range(total_user_records, total_user_records + records_remaining_this_loop)) for purchases in carts: for purchase in purchases: total_purchase_records += 1 @@ -288,7 +165,6 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: user_records_in_sync += 1 if user_records_in_sync == self.records_per_sync: - running = False break self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py new file mode 100644 index 000000000000..6fab4dc2bb48 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from multiprocessing import current_process + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Person +from mimesis.locales import Locale + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class UserGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same thread, resetting the internal random counters + """ + + seed_with_offset = self.seed + if len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global person + global dt + + person = Person(locale=Locale.EN, seed=seed_with_offset) + dt = Datetime(seed=seed_with_offset) + + def generate(self, user_id: int): + time_a = dt.datetime() + time_b = dt.datetime() + + # faker doesn't always produce unique email addresses, so to enforce uniqueness, we will append the user_id to the prefix + email_parts = person.email().split("@") + email = f"{email_parts[0]}+{user_id + 1}@{email_parts[1]}" + + profile = { + "id": user_id + 1, + "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), + "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), + "name": person.name(), + "title": person.title(), + "age": person.age(), + "email": email, + "telephone": person.telephone(), + "gender": person.gender(), + "language": person.language(), + "academic_degree": person.academic_degree(), + "nationality": person.nationality(), + "occupation": person.occupation(), + "height": person.height(), + "blood_type": person.blood_type(), + "weight": person.weight(), + } + + while not profile["created_at"]: + profile["created_at"] = format_airbyte_time(dt.datetime()) + + if not profile["updated_at"]: + profile["updated_at"] = profile["created_at"] + 1 + + record = AirbyteRecordMessage(stream=self.stream_name, data=profile, emitted_at=now_millis()) + return AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) From 1def34c2c90d7e736ab74e8e2fd3278512811f2c Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 22 Dec 2022 13:47:45 -0800 Subject: [PATCH 10/13] more comments --- .../connectors/source-faker/source_faker/streams.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index fcc7b9a9f9dc..089bc43e5f6d 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -88,6 +88,11 @@ def state(self, value: Mapping[str, Any]): self._state = value def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ + total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 records_in_sync = 0 @@ -142,6 +147,11 @@ def state(self, value: Mapping[str, Any]): self._state = value def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ + total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 total_user_records = self.state["user_id"] if "user_id" in self.state else 0 user_records_in_sync = 0 From ebbe63e313fb565387c31841e24daddc2cba4bbd Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 22 Dec 2022 14:39:25 -0800 Subject: [PATCH 11/13] `parallelism` and pass tests --- .../source_faker/purchase_generator.py | 2 +- .../source-faker/source_faker/source.py | 8 +++---- .../source-faker/source_faker/spec.json | 6 ++--- .../source-faker/source_faker/streams.py | 22 +++++++++---------- .../source_faker/user_generator.py | 2 +- .../source-faker/unit_tests/unit_test.py | 22 +++++++++---------- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py index b1a206a33bf8..3caa19ddf788 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -27,7 +27,7 @@ def prepare(self): """ seed_with_offset = self.seed - if len(current_process()._identity) > 0: + if self.seed is not None and len(current_process()._identity) > 0: seed_with_offset = self.seed + current_process()._identity[0] global dt diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index a8b877951861..3772cea75516 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -23,10 +23,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: seed: int = config["seed"] if "seed" in config else None records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500 records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100 - threads: int = config["threads"] if "threads" in config else 4 + parallelism: int = config["parallelism"] if "parallelism" in config else 4 return [ - Products(count, seed, threads, records_per_sync, records_per_slice), - Users(count, seed, threads, records_per_sync, records_per_slice), - Purchases(count, seed, threads, records_per_sync, records_per_slice), + Products(count, seed, parallelism, records_per_sync, records_per_slice), + Users(count, seed, parallelism, records_per_sync, records_per_slice), + Purchases(count, seed, parallelism, records_per_sync, records_per_slice), ] diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index 0715a4ac366c..0d20f791000d 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -38,9 +38,9 @@ "default": 1000, "order": 3 }, - "threads": { - "title": "Worker Threads", - "description": "How many parallel worker threads should be used to generate records?", + "parallelism": { + "title": "Parallelism", + "description": "How many parallel workers should we use to generate fake data? Choose a value equal to the number of CPUs you will allocate to this source.", "type": "integer", "minimum": 1, "default": 4, diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 089bc43e5f6d..a25e93813615 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -17,7 +17,7 @@ class Products(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.seed = seed self.records_per_sync = records_per_sync @@ -63,13 +63,13 @@ class Users(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.threads = threads + self.parallelism = parallelism self.generator = UserGenerator(self.name, self.seed) @property @@ -99,8 +99,8 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: median_record_byte_size = 450 yield generate_estimate(self.name, self.count - total_records, median_record_byte_size) - with Pool(initializer=self.generator.prepare, processes=self.threads) as pool: - while records_in_sync < self.count: + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while records_in_sync < self.count and records_in_sync < self.records_per_sync: records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) if records_remaining_this_loop <= 0: break @@ -110,7 +110,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: records_in_sync += 1 yield user - if records_in_sync == self.records_per_sync: + if records_in_sync >= self.records_per_sync: break self.state = {self.cursor_field: total_records, "seed": self.seed} @@ -122,13 +122,13 @@ class Purchases(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, threads: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.threads = threads + self.parallelism = parallelism self.generator = PurchaseGenerator(self.name, self.seed) @property @@ -160,8 +160,8 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: median_record_byte_size = 230 yield generate_estimate(self.name, (self.count - total_user_records) * 1.3, median_record_byte_size) - with Pool(initializer=self.generator.prepare, processes=self.threads) as pool: - while total_user_records < self.count: + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while total_user_records < self.count and user_records_in_sync < self.records_per_sync: records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) if records_remaining_this_loop <= 0: break @@ -174,7 +174,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: total_user_records += 1 user_records_in_sync += 1 - if user_records_in_sync == self.records_per_sync: + if user_records_in_sync >= self.records_per_sync: break self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py index 6fab4dc2bb48..a743456816fc 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -26,7 +26,7 @@ def prepare(self): """ seed_with_offset = self.seed - if len(current_process()._identity) > 0: + if self.seed is not None and len(current_process()._identity) > 0: seed_with_offset = self.seed + current_process()._identity[0] global person diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index 2a21bd69530a..193960b73453 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -25,7 +25,7 @@ def exception(a,b,**kwargs): def schemas_are_valid(): source = SourceFaker() - config = {"count": 1, "threads": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -36,7 +36,7 @@ def schemas_are_valid(): def test_source_streams(): source = SourceFaker() - config = {"count": 1, "threads": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -64,7 +64,7 @@ def test_source_streams(): def test_read_small_random_data(): source = SourceFaker() - config = {"count": 10, "threads": 1} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -98,7 +98,7 @@ def test_read_small_random_data(): def test_no_read_limit_hit(): source = SourceFaker() - config = {"count": 10, "threads": 1} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -128,7 +128,7 @@ def test_no_read_limit_hit(): def test_read_big_random_data(): source = SourceFaker() - config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000, "threads": 1} + config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -163,7 +163,7 @@ def test_read_big_random_data(): def test_with_purchases(): source = SourceFaker() - config = {"count": 1000, "records_per_sync": 1000, "threads": 1} + config = {"count": 1000, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -205,7 +205,7 @@ def test_with_purchases(): def test_sync_ends_with_limit(): source = SourceFaker() - config = {"count": 100, "records_per_sync": 5, "threads": 1} + config = {"count": 100, "records_per_sync": 5, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -239,7 +239,7 @@ def test_read_with_seed(): """ source = SourceFaker() - config = {"count": 1, "seed": 100, "threads": 1} + config = {"count": 1, "seed": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -253,14 +253,14 @@ def test_read_with_seed(): iterator = source.read(logger, config, catalog, state) records = [row for row in iterator if row.type is Type.RECORD] - assert records[0].record.data["occupation"] == "Illustrator" - assert records[0].record.data["email"] == "upcoming1928@protonmail.com" + assert records[0].record.data["occupation"] == "Cartoonist" + assert records[0].record.data["email"] == "reflect1958+1@yahoo.com" def test_ensure_no_purchases_without_users(): with pytest.raises(ValueError): source = SourceFaker() - config = {"count": 100, "threads": 1} + config = {"count": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ {"stream": {"name": "purchases", "json_schema": {}}, "sync_mode": "incremental", "destination_sync_mode": "overwrite"}, From 310605b04e5bcbe605934bce9094399beb7e3523 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 22 Dec 2022 14:42:57 -0800 Subject: [PATCH 12/13] update expected records --- .../integration_tests/expected_records.jsonl | 240 +++++++++--------- 1 file changed, 120 insertions(+), 120 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index 193e54ef3713..e6e1c3e60c9a 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -1,120 +1,120 @@ -{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Walter", "title": "PhD", "age": 47, "email": "similarly1907@example.com", "telephone": "084.704.7299", "gender": "Male", "language": "Amharic", "academic_degree": "Bachelor", "nationality": "Saudi", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1671665757124} -{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Nestor", "title": "Sir", "age": 47, "email": "guys1918@example.org", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Arnita", "title": "Madam", "age": 42, "email": "painted1991@yandex.com", "telephone": "611.837.4386", "gender": "Other", "language": "Spanish", "academic_degree": "Bachelor", "nationality": "Chinese", "occupation": "Textile Engineer", "height": "1.61", "blood_type": "O+", "weight": 87}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 4, "created_at": "2000-09-08T14:31:35+00:00", "updated_at": "2011-04-22T07:48:29+00:00", "name": "Theron", "title": "M.Sc.Tech.", "age": 38, "email": "sectors2080@outlook.com", "telephone": "1-788-785-5408", "gender": "Female", "language": "Montenegrin", "academic_degree": "Master", "nationality": "Japanese", "occupation": "Printer", "height": "1.71", "blood_type": "O+", "weight": 72}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 5, "created_at": "2005-11-24T09:07:47+00:00", "updated_at": "2009-01-14T17:59:41+00:00", "name": "Sharika", "title": "Master", "age": 27, "email": "espn2082@example.org", "telephone": "860-007-2163", "gender": "Male", "language": "Dutch", "academic_degree": "Bachelor", "nationality": "Ecuadorian", "occupation": "Travel Courier", "height": "1.98", "blood_type": "O\u2212", "weight": 79}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 6, "created_at": "2021-04-10T09:37:56+00:00", "updated_at": "2022-09-30T13:32:53+00:00", "name": "Shanita", "title": "MSc", "age": 23, "email": "mali1975@duck.com", "telephone": "227.994.9230", "gender": "Female", "language": "Malagasy", "academic_degree": "PhD", "nationality": "Jordanian", "occupation": "Security Consultant", "height": "1.77", "blood_type": "AB+", "weight": 78}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 7, "created_at": "2012-07-22T05:23:35+00:00", "updated_at": "2016-07-19T01:30:15+00:00", "name": "Jen", "title": "LL.D", "age": 58, "email": "blogs1864@protonmail.com", "telephone": "383-974-2080", "gender": "Female", "language": "Pashto", "academic_degree": "Master", "nationality": "Chinese", "occupation": "Timber Worker", "height": "1.52", "blood_type": "AB+", "weight": 60}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 8, "created_at": "2016-02-25T05:33:53+00:00", "updated_at": "2022-11-24T11:05:28+00:00", "name": "Fermin", "title": "Madam", "age": 22, "email": "letting2055@protonmail.com", "telephone": "(642) 188-5142", "gender": "Male", "language": "Khmer", "academic_degree": "PhD", "nationality": "Finnish", "occupation": "Turkey Farmer", "height": "1.74", "blood_type": "AB+", "weight": 42}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 9, "created_at": "2011-08-24T00:30:02+00:00", "updated_at": "2022-10-19T18:25:41+00:00", "name": "Cecil", "title": "Mr.", "age": 50, "email": "interference2059@yahoo.com", "telephone": "(312) 977-4573", "gender": "Other", "language": "Bengali", "academic_degree": "Master", "nationality": "Chilean", "occupation": "Foreman", "height": "1.67", "blood_type": "B\u2212", "weight": 42}, "emitted_at": 1671665757125} -{"stream": "users", "data": {"id": 10, "created_at": "2005-09-08T00:49:12+00:00", "updated_at": "2017-04-13T16:22:54+00:00", "name": "Von", "title": "M.A.", "age": 34, "email": "consequently1941@example.org", "telephone": "806-018-0294", "gender": "Male", "language": "Malagasy", "academic_degree": "Bachelor", "nationality": "Spanish", "occupation": "Machinist", "height": "1.58", "blood_type": "A+", "weight": 65}, "emitted_at": 1671665757125} -{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2020-01-12T11:53:10+00:00", "purchased_at": "2022-03-10T11:53:10+00:00", "returned_at": "2022-10-11T11:53:10+00:00"}, "emitted_at": 1671665757354} -{"stream": "purchases", "data": {"id": 2, "product_id": 47, "user_id": 2, "added_to_cart_at": "2016-11-30T14:32:17+00:00", "purchased_at": "2019-10-26T14:32:17+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 3, "product_id": 86, "user_id": 3, "added_to_cart_at": "2022-09-08T14:59:20+00:00", "purchased_at": "2022-11-12T14:59:20+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 4, "product_id": 78, "user_id": 4, "added_to_cart_at": "2015-11-20T05:20:11+00:00", "purchased_at": "2020-08-07T05:20:11+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 5, "product_id": 5, "user_id": 5, "added_to_cart_at": "2012-03-25T11:54:28+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 6, "product_id": 88, "user_id": 6, "added_to_cart_at": "2015-11-19T14:41:33+00:00", "purchased_at": "2021-08-13T14:41:33+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 7, "product_id": 82, "user_id": 8, "added_to_cart_at": "2021-04-12T17:46:29+00:00", "purchased_at": "2022-05-11T17:46:29+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 8, "product_id": 66, "user_id": 9, "added_to_cart_at": "2021-10-02T15:19:19+00:00", "purchased_at": "2021-10-22T15:19:19+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 9, "product_id": 57, "user_id": 9, "added_to_cart_at": "2019-12-22T23:13:31+00:00", "purchased_at": "2022-11-03T23:13:31+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "purchases", "data": {"id": 10, "product_id": 5, "user_id": 10, "added_to_cart_at": "2017-08-04T03:03:36+00:00", "purchased_at": "2021-11-03T03:03:36+00:00", "returned_at": null}, "emitted_at": 1671665757355} -{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1671665757360} -{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1671665757361} -{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1671665757362} -{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1671665757363} -{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1671665757363} +{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Rudolf", "title": "M.Des", "age": 66, "email": "wisconsin1930+1@yandex.com", "telephone": "(483) 676-2851", "gender": "Fluid", "language": "Arabic", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Orville", "title": "Miss", "age": 30, "email": "recipes2070+2@yahoo.com", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Rachell", "title": "M.A.", "age": 21, "email": "assets1924+3@protonmail.com", "telephone": "+1-(118)-374-3865", "gender": "Female", "language": "Dutch", "academic_degree": "PhD", "nationality": "Danish", "occupation": "Aeronautical Engineer", "height": "1.89", "blood_type": "AB+", "weight": 63}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 4, "created_at": "2000-09-08T14:31:35+00:00", "updated_at": "2011-04-22T07:48:29+00:00", "name": "Yer", "title": "M.Sc.Tech.", "age": 24, "email": "necessary2035+4@example.org", "telephone": "+1-(294)-359-4840", "gender": "Fluid", "language": "Malay", "academic_degree": "PhD", "nationality": "Argentinian", "occupation": "Line Manager", "height": "1.82", "blood_type": "B+", "weight": 43}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 5, "created_at": "2005-11-24T09:07:47+00:00", "updated_at": "2009-01-14T17:59:41+00:00", "name": "Alton", "title": "Miss", "age": 31, "email": "implementing1836+5@example.org", "telephone": "(712) 129-6627", "gender": "Other", "language": "Maltese", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Paint Consultant", "height": "1.69", "blood_type": "B\u2212", "weight": 88}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 6, "created_at": "2021-04-10T09:37:56+00:00", "updated_at": "2022-09-30T13:32:53+00:00", "name": "Octavio", "title": "Mr.", "age": 41, "email": "ind1929+6@yahoo.com", "telephone": "717-652-9752", "gender": "Male", "language": "Kurdish", "academic_degree": "PhD", "nationality": "Polish", "occupation": "Pathologist", "height": "1.83", "blood_type": "B\u2212", "weight": 41}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 7, "created_at": "2012-07-22T05:23:35+00:00", "updated_at": "2016-07-19T01:30:15+00:00", "name": "Casimira", "title": "B.Sc", "age": 63, "email": "coupled1824+7@live.com", "telephone": "1-515-852-9488", "gender": "Male", "language": "Khmer", "academic_degree": "Bachelor", "nationality": "Finnish", "occupation": "Chicken Chaser", "height": "1.60", "blood_type": "B\u2212", "weight": 75}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 8, "created_at": "2016-02-25T05:33:53+00:00", "updated_at": "2022-11-24T11:05:28+00:00", "name": "Terrell", "title": "Miss", "age": 55, "email": "stick1999+8@yahoo.com", "telephone": "023-973-2689", "gender": "Fluid", "language": "Tamil", "academic_degree": "PhD", "nationality": "Japanese", "occupation": "Trout Farmer", "height": "1.62", "blood_type": "O+", "weight": 43}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 9, "created_at": "2011-08-24T00:30:02+00:00", "updated_at": "2022-10-19T18:25:41+00:00", "name": "Ira", "title": "M.A.", "age": 37, "email": "efforts2075+9@yandex.com", "telephone": "(286) 981-5100", "gender": "Female", "language": "Croatian", "academic_degree": "Master", "nationality": "Cameroonian", "occupation": "Purchase Clerk", "height": "1.54", "blood_type": "AB+", "weight": 74}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 10, "created_at": "2005-09-08T00:49:12+00:00", "updated_at": "2017-04-13T16:22:54+00:00", "name": "Randall", "title": "Mrs.", "age": 34, "email": "intellectual1951+10@gmail.com", "telephone": "018-029-4112", "gender": "Male", "language": "Luxembourgish", "academic_degree": "Master", "nationality": "Mexican", "occupation": "Nursery Nurse", "height": "1.78", "blood_type": "AB\u2212", "weight": 58}, "emitted_at": 1671748915185} +{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2003-02-23T11:53:10+00:00", "purchased_at": "2011-03-30T11:53:10+00:00", "returned_at": null}, "emitted_at": 1671748915424} +{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "added_to_cart_at": "2019-12-02T14:32:17+00:00", "purchased_at": "2021-02-08T14:32:17+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 3, "product_id": 5, "user_id": 3, "added_to_cart_at": "2014-01-25T14:59:20+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 4, "product_id": 21, "user_id": 4, "added_to_cart_at": "2016-11-27T05:20:11+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 5, "product_id": 51, "user_id": 5, "added_to_cart_at": "2020-10-15T11:54:28+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 6, "product_id": 66, "user_id": 6, "added_to_cart_at": "2017-02-24T14:41:33+00:00", "purchased_at": "2022-02-20T14:41:33+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 7, "product_id": 35, "user_id": 8, "added_to_cart_at": "2021-12-01T17:46:29+00:00", "purchased_at": "2021-12-15T17:46:29+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 8, "product_id": 60, "user_id": 9, "added_to_cart_at": "2021-03-07T15:19:19+00:00", "purchased_at": "2022-05-14T15:19:19+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 9, "product_id": 22, "user_id": 9, "added_to_cart_at": "2021-08-30T23:13:31+00:00", "purchased_at": "2021-12-28T23:13:31+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 10, "product_id": 4, "user_id": 10, "added_to_cart_at": "2008-03-28T03:03:36+00:00", "purchased_at": "2012-02-18T03:03:36+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1671748915434} From 64a5045798fd8bcfb8ffd952f5ba5a25ff837191 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 22 Dec 2022 14:44:15 -0800 Subject: [PATCH 13/13] cleanup notes --- .../source_faker/airbyte_message_with_cached_json.py | 2 +- .../connectors/source-faker/source_faker/purchase_generator.py | 2 +- .../connectors/source-faker/source_faker/user_generator.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py index 316cbe76405f..be338ddb4afe 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py @@ -8,7 +8,7 @@ class AirbyteMessageWithCachedJSON(AirbyteMessage): """ I a monkeypatch to AirbyteMessage which pre-renders the JSON-representation of the object upon initialization. - This allows the JSON to be calculated in the thread/process that builds the object rather than the main thread. + This allows the JSON to be calculated in the process that builds the object rather than the main process. Note: We can't use @cache here because the LRU cache is not serializable when passed to child workers. """ diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py index 3caa19ddf788..53cf05be4177 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -23,7 +23,7 @@ def prepare(self): Note: the instances of the mimesis generators need to be global. Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork - * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same thread, resetting the internal random counters + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters """ seed_with_offset = self.seed diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py index a743456816fc..3fcedd2b6ad4 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -22,7 +22,7 @@ def prepare(self): Note: the instances of the mimesis generators need to be global. Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork - * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same thread, resetting the internal random counters + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters """ seed_with_offset = self.seed