diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index 87ff4c896424..e6e1c3e60c9a 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -1,123 +1,120 @@ -{"stream": "users", "data": {"id": 1, "created_at": "2009-08-12T18:57:58+00:00", "updated_at": "2012-07-02T08:32:31+00:00", "name": "Reda", "title": "M.Sc.Tech.", "age": 47, "email": "locations1983@protonmail.com", "telephone": "+1-(110)-795-7610", "gender": "Male", "language": "Tamil", "academic_degree": "Master", "nationality": "Italian", "occupation": "Word Processing Operator", "height": "1.55", "blood_type": "B\u2212", "weight": 58}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 2, "created_at": "2008-09-23T19:57:09+00:00", "updated_at": "2016-03-10T04:48:06+00:00", "name": "Tristan", "title": "M.Sc.Tech.", "age": 32, "email": "variations1847@duck.com", "telephone": "683-770-9281", "gender": "Other", "language": "Bosnian", "academic_degree": "Bachelor", "nationality": "Estonian", "occupation": "Tiler", "height": "2.00", "blood_type": "AB\u2212", "weight": 44}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 3, "created_at": "2003-06-14T10:39:40+00:00", "updated_at": "2003-12-03T21:21:30+00:00", "name": "Yuki", "title": "Miss", "age": 50, "email": "vacuum2027@yahoo.com", "telephone": "1-321-809-2061", "gender": "Female", "language": "Armenian", "academic_degree": "Bachelor", "nationality": "Swiss", "occupation": "Valuer", "height": "1.84", "blood_type": "O\u2212", "weight": 71}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 4, "created_at": "2001-09-30T00:05:46+00:00", "updated_at": "2006-09-16T14:55:33+00:00", "name": "Fred", "title": "MMath", "age": 47, "email": "causes1859@outlook.com", "telephone": "(827) 127-3811", "gender": "Female", "language": "Assamese", "academic_degree": "PhD", "nationality": "Russian", "occupation": "Turkey Farmer", "height": "1.80", "blood_type": "A+", "weight": 39}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 5, "created_at": "2012-12-27T21:40:00+00:00", "updated_at": "2015-06-08T23:20:45+00:00", "name": "Emmitt", "title": "DPhil", "age": 39, "email": "athens1899@gmail.com", "telephone": "(470) 656-8003", "gender": "Other", "language": "English", "academic_degree": "Bachelor", "nationality": "Jordanian", "occupation": "Stone Sawyer", "height": "1.52", "blood_type": "A+", "weight": 82}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 6, "created_at": "2002-04-30T18:14:15+00:00", "updated_at": "2004-09-15T02:05:20+00:00", "name": "Hollis", "title": "MSc", "age": 52, "email": "fisheries1881@yandex.com", "telephone": "(519) 606-9896", "gender": "Other", "language": "Swati", "academic_degree": "Master", "nationality": "Chilean", "occupation": "Writer", "height": "1.85", "blood_type": "AB\u2212", "weight": 85}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 7, "created_at": "2003-09-11T17:13:51+00:00", "updated_at": "2016-08-04T09:35:18+00:00", "name": "Kip", "title": "M.D.", "age": 31, "email": "numbers1983@example.com", "telephone": "346-013-2638", "gender": "Other", "language": "Armenian", "academic_degree": "Master", "nationality": "Swiss", "occupation": "Salesman", "height": "1.89", "blood_type": "O+", "weight": 48}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 8, "created_at": "2012-06-19T07:18:11+00:00", "updated_at": "2017-10-10T14:05:38+00:00", "name": "Carie", "title": "Madam", "age": 49, "email": "watershed1819@example.com", "telephone": "(348) 881-9607", "gender": "Male", "language": "Kyrgyz", "academic_degree": "PhD", "nationality": "Guatemalan", "occupation": "Park Ranger", "height": "1.50", "blood_type": "O+", "weight": 83}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 9, "created_at": "2002-11-25T04:56:09+00:00", "updated_at": "2005-01-20T21:16:30+00:00", "name": "Steven", "title": "Mr.", "age": 54, "email": "llp1893@yahoo.com", "telephone": "830.247.8156", "gender": "Fluid", "language": "Catalan", "academic_degree": "Bachelor", "nationality": "Egyptian", "occupation": "Ambulance Driver", "height": "1.52", "blood_type": "AB+", "weight": 81}, "emitted_at": 1669830193008} -{"stream": "users", "data": {"id": 10, "created_at": "2001-02-23T17:43:25+00:00", "updated_at": "2022-09-09T16:51:15+00:00", "name": "Lore", "title": "Madam", "age": 61, "email": "resident2075@example.com", "telephone": "321.233.0702", "gender": "Female", "language": "Polish", "academic_degree": "Master", "nationality": "French", "occupation": "Registrar", "height": "1.99", "blood_type": "B+", "weight": 56}, "emitted_at": 1669830193008} -{"stream": "purchases", "data": {"id": 1, "product_id": 98, "user_id": 1, "added_to_cart_at": "2019-01-17T18:57:58+00:00", "purchased_at": "2020-06-30T18:57:58+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 2, "product_id": 39, "user_id": 2, "added_to_cart_at": "2019-06-02T19:57:09+00:00", "purchased_at": "2022-09-08T19:57:09+00:00", "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 3, "product_id": 37, "user_id": 3, "added_to_cart_at": "2006-08-01T10:39:40+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193009} -{"stream": "purchases", "data": {"id": 4, "product_id": 80, "user_id": 3, "added_to_cart_at": "2021-05-18T10:39:40+00:00", "purchased_at": "2022-11-14T10:39:40+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 5, "product_id": 40, "user_id": 4, "added_to_cart_at": "2003-12-18T00:05:46+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 6, "product_id": 88, "user_id": 4, "added_to_cart_at": "2009-02-24T00:05:46+00:00", "purchased_at": "2021-09-14T00:05:46+00:00", "returned_at": "2022-03-14T00:05:46+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 7, "product_id": 79, "user_id": 5, "added_to_cart_at": "2020-03-03T21:40:00+00:00", "purchased_at": "2022-11-17T21:40:00+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 8, "product_id": 67, "user_id": 6, "added_to_cart_at": "2008-03-02T18:14:15+00:00", "purchased_at": "2020-06-21T18:14:15+00:00", "returned_at": "2020-09-24T18:14:15+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 9, "product_id": 91, "user_id": 7, "added_to_cart_at": "2022-03-12T17:13:51+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 10, "product_id": 79, "user_id": 8, "added_to_cart_at": "2017-12-31T07:18:11+00:00", "purchased_at": "2019-05-14T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 11, "product_id": 91, "user_id": 8, "added_to_cart_at": "2022-03-24T07:18:11+00:00", "purchased_at": "2022-06-29T07:18:11+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 12, "product_id": 19, "user_id": 9, "added_to_cart_at": "2020-11-29T04:56:09+00:00", "purchased_at": "2022-03-02T04:56:09+00:00", "returned_at": "2022-04-12T04:56:09+00:00"}, "emitted_at": 1669830193010} -{"stream": "purchases", "data": {"id": 13, "product_id": 63, "user_id": 10, "added_to_cart_at": "2003-08-05T17:43:25+00:00", "purchased_at": "2015-12-15T17:43:25+00:00", "returned_at": null}, "emitted_at": 1669830193010} -{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1669830193011} -{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1669830193012} -{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1669830193013} -{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1669830193014} -{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1669830193015} -{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1669830193015} +{"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Rudolf", "title": "M.Des", "age": 66, "email": "wisconsin1930+1@yandex.com", "telephone": "(483) 676-2851", "gender": "Fluid", "language": "Arabic", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Orville", "title": "Miss", "age": 30, "email": "recipes2070+2@yahoo.com", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Rachell", "title": "M.A.", "age": 21, "email": "assets1924+3@protonmail.com", "telephone": "+1-(118)-374-3865", "gender": "Female", "language": "Dutch", "academic_degree": "PhD", "nationality": "Danish", "occupation": "Aeronautical Engineer", "height": "1.89", "blood_type": "AB+", "weight": 63}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 4, "created_at": "2000-09-08T14:31:35+00:00", "updated_at": "2011-04-22T07:48:29+00:00", "name": "Yer", "title": "M.Sc.Tech.", "age": 24, "email": "necessary2035+4@example.org", "telephone": "+1-(294)-359-4840", "gender": "Fluid", "language": "Malay", "academic_degree": "PhD", "nationality": "Argentinian", "occupation": "Line Manager", "height": "1.82", "blood_type": "B+", "weight": 43}, "emitted_at": 1671748915184} +{"stream": "users", "data": {"id": 5, "created_at": "2005-11-24T09:07:47+00:00", "updated_at": "2009-01-14T17:59:41+00:00", "name": "Alton", "title": "Miss", "age": 31, "email": "implementing1836+5@example.org", "telephone": "(712) 129-6627", "gender": "Other", "language": "Maltese", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Paint Consultant", "height": "1.69", "blood_type": "B\u2212", "weight": 88}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 6, "created_at": "2021-04-10T09:37:56+00:00", "updated_at": "2022-09-30T13:32:53+00:00", "name": "Octavio", "title": "Mr.", "age": 41, "email": "ind1929+6@yahoo.com", "telephone": "717-652-9752", "gender": "Male", "language": "Kurdish", "academic_degree": "PhD", "nationality": "Polish", "occupation": "Pathologist", "height": "1.83", "blood_type": "B\u2212", "weight": 41}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 7, "created_at": "2012-07-22T05:23:35+00:00", "updated_at": "2016-07-19T01:30:15+00:00", "name": "Casimira", "title": "B.Sc", "age": 63, "email": "coupled1824+7@live.com", "telephone": "1-515-852-9488", "gender": "Male", "language": "Khmer", "academic_degree": "Bachelor", "nationality": "Finnish", "occupation": "Chicken Chaser", "height": "1.60", "blood_type": "B\u2212", "weight": 75}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 8, "created_at": "2016-02-25T05:33:53+00:00", "updated_at": "2022-11-24T11:05:28+00:00", "name": "Terrell", "title": "Miss", "age": 55, "email": "stick1999+8@yahoo.com", "telephone": "023-973-2689", "gender": "Fluid", "language": "Tamil", "academic_degree": "PhD", "nationality": "Japanese", "occupation": "Trout Farmer", "height": "1.62", "blood_type": "O+", "weight": 43}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 9, "created_at": "2011-08-24T00:30:02+00:00", "updated_at": "2022-10-19T18:25:41+00:00", "name": "Ira", "title": "M.A.", "age": 37, "email": "efforts2075+9@yandex.com", "telephone": "(286) 981-5100", "gender": "Female", "language": "Croatian", "academic_degree": "Master", "nationality": "Cameroonian", "occupation": "Purchase Clerk", "height": "1.54", "blood_type": "AB+", "weight": 74}, "emitted_at": 1671748915185} +{"stream": "users", "data": {"id": 10, "created_at": "2005-09-08T00:49:12+00:00", "updated_at": "2017-04-13T16:22:54+00:00", "name": "Randall", "title": "Mrs.", "age": 34, "email": "intellectual1951+10@gmail.com", "telephone": "018-029-4112", "gender": "Male", "language": "Luxembourgish", "academic_degree": "Master", "nationality": "Mexican", "occupation": "Nursery Nurse", "height": "1.78", "blood_type": "AB\u2212", "weight": 58}, "emitted_at": 1671748915185} +{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2003-02-23T11:53:10+00:00", "purchased_at": "2011-03-30T11:53:10+00:00", "returned_at": null}, "emitted_at": 1671748915424} +{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "added_to_cart_at": "2019-12-02T14:32:17+00:00", "purchased_at": "2021-02-08T14:32:17+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 3, "product_id": 5, "user_id": 3, "added_to_cart_at": "2014-01-25T14:59:20+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 4, "product_id": 21, "user_id": 4, "added_to_cart_at": "2016-11-27T05:20:11+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 5, "product_id": 51, "user_id": 5, "added_to_cart_at": "2020-10-15T11:54:28+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 6, "product_id": 66, "user_id": 6, "added_to_cart_at": "2017-02-24T14:41:33+00:00", "purchased_at": "2022-02-20T14:41:33+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 7, "product_id": 35, "user_id": 8, "added_to_cart_at": "2021-12-01T17:46:29+00:00", "purchased_at": "2021-12-15T17:46:29+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 8, "product_id": 60, "user_id": 9, "added_to_cart_at": "2021-03-07T15:19:19+00:00", "purchased_at": "2022-05-14T15:19:19+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 9, "product_id": 22, "user_id": 9, "added_to_cart_at": "2021-08-30T23:13:31+00:00", "purchased_at": "2021-12-28T23:13:31+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "purchases", "data": {"id": 10, "product_id": 4, "user_id": 10, "added_to_cart_at": "2008-03-28T03:03:36+00:00", "purchased_at": "2012-02-18T03:03:36+00:00", "returned_at": null}, "emitted_at": 1671748915425} +{"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 4, "make": "GMC", "model": "Jimmy", "year": 1998, "price": 34079, "created_at": "2022-01-24T03:00:03+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 5, "make": "Infiniti", "model": "FX", "year": 2004, "price": 17036, "created_at": "2021-10-02T03:55:44+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 6, "make": "Dodge", "model": "Intrepid", "year": 2002, "price": 65498, "created_at": "2022-01-18T00:41:08+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 7, "make": "Nissan", "model": "Frontier", "year": 2005, "price": 14516, "created_at": "2021-04-22T16:37:44+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 8, "make": "Chevrolet", "model": "Express 1500", "year": 2007, "price": 13023, "created_at": "2021-07-12T07:13:04+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 9, "make": "Bentley", "model": "Continental GTC", "year": 2008, "price": 43458, "created_at": "2021-03-17T05:43:15+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 10, "make": "Cadillac", "model": "DTS", "year": 2008, "price": 43859, "created_at": "2021-08-12T07:33:58+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 11, "make": "Dodge", "model": "Ram 2500", "year": 2000, "price": 82904, "created_at": "2021-09-03T10:51:16+00:00"}, "emitted_at": 1671748915430} +{"stream": "products", "data": {"id": 12, "make": "Suzuki", "model": "SJ 410", "year": 1984, "price": 38667, "created_at": "2021-01-11T00:15:46+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 13, "make": "Audi", "model": "S4", "year": 2005, "price": 2391, "created_at": "2021-09-06T03:31:10+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 14, "make": "Chevrolet", "model": "Suburban 2500", "year": 1998, "price": 55733, "created_at": "2021-10-18T17:26:05+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 15, "make": "Ford", "model": "Ranger", "year": 2000, "price": 20228, "created_at": "2022-03-24T04:03:19+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 16, "make": "Chevrolet", "model": "Corvette", "year": 2009, "price": 75052, "created_at": "2021-12-31T03:38:21+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 17, "make": "Mitsubishi", "model": "Pajero", "year": 1993, "price": 84058, "created_at": "2021-10-15T00:25:34+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 18, "make": "Lincoln", "model": "LS", "year": 2002, "price": 34081, "created_at": "2022-02-14T22:12:01+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 19, "make": "Dodge", "model": "Magnum", "year": 2005, "price": 85545, "created_at": "2021-07-25T22:49:48+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 20, "make": "Pontiac", "model": "Grand Am", "year": 2001, "price": 54837, "created_at": "2021-10-15T14:08:30+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 21, "make": "Chevrolet", "model": "Suburban 1500", "year": 2006, "price": 89410, "created_at": "2021-03-23T15:40:43+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 22, "make": "GMC", "model": "Sierra 1500", "year": 2005, "price": 14288, "created_at": "2021-08-30T13:40:04+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 23, "make": "GMC", "model": "3500", "year": 1995, "price": 12011, "created_at": "2022-04-24T13:11:08+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 24, "make": "Mazda", "model": "Mazda5", "year": 2006, "price": 6393, "created_at": "2021-07-07T14:14:33+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 25, "make": "Chevrolet", "model": "Camaro", "year": 1967, "price": 71590, "created_at": "2021-01-10T21:50:22+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 26, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 23498, "created_at": "2022-04-20T00:52:20+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 27, "make": "Dodge", "model": "Caravan", "year": 1985, "price": 50071, "created_at": "2022-01-05T10:13:31+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 28, "make": "Nissan", "model": "240SX", "year": 1992, "price": 38379, "created_at": "2022-04-07T04:48:48+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 29, "make": "Oldsmobile", "model": "Intrigue", "year": 2002, "price": 21376, "created_at": "2021-10-01T13:30:49+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 30, "make": "Audi", "model": "TT", "year": 2011, "price": 40893, "created_at": "2021-02-28T23:06:37+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 31, "make": "Ford", "model": "Crown Victoria", "year": 2006, "price": 86225, "created_at": "2021-01-28T23:33:27+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 32, "make": "Toyota", "model": "Tacoma", "year": 2003, "price": 73558, "created_at": "2022-01-28T22:02:04+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 33, "make": "Buick", "model": "Regal", "year": 1994, "price": 32279, "created_at": "2022-04-04T13:35:49+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 34, "make": "Mercedes-Benz", "model": "C-Class", "year": 2001, "price": 98732, "created_at": "2021-03-30T23:16:05+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 35, "make": "GMC", "model": "Sierra 3500", "year": 2002, "price": 48267, "created_at": "2021-07-30T20:29:51+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 36, "make": "Pontiac", "model": "G6", "year": 2005, "price": 16766, "created_at": "2021-03-24T07:53:33+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 37, "make": "Subaru", "model": "Outback Sport", "year": 2002, "price": 34523, "created_at": "2021-12-23T22:47:32+00:00"}, "emitted_at": 1671748915431} +{"stream": "products", "data": {"id": 38, "make": "Ferrari", "model": "F430", "year": 2007, "price": 31677, "created_at": "2021-01-11T04:49:57+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 39, "make": "Mitsubishi", "model": "Montero", "year": 2003, "price": 67136, "created_at": "2021-05-10T07:37:56+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 40, "make": "Nissan", "model": "Sentra", "year": 1993, "price": 78236, "created_at": "2021-11-10T23:48:26+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 41, "make": "Mitsubishi", "model": "3000GT", "year": 1993, "price": 58150, "created_at": "2021-09-08T06:55:22+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 42, "make": "Ford", "model": "E350", "year": 2012, "price": 55270, "created_at": "2021-03-24T13:17:37+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 43, "make": "Ford", "model": "Taurus", "year": 1987, "price": 13522, "created_at": "2021-10-27T21:03:59+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 44, "make": "Chevrolet", "model": "Avalanche", "year": 2012, "price": 9862, "created_at": "2021-07-13T12:22:26+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 45, "make": "Dodge", "model": "Charger", "year": 2012, "price": 81887, "created_at": "2021-04-24T01:48:24+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 46, "make": "Jaguar", "model": "S-Type", "year": 2005, "price": 34372, "created_at": "2021-04-03T08:56:17+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 47, "make": "Plymouth", "model": "Grand Voyager", "year": 1994, "price": 90637, "created_at": "2022-04-21T09:21:08+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 48, "make": "Pontiac", "model": "6000", "year": 1989, "price": 65165, "created_at": "2021-10-30T13:03:07+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 49, "make": "Lexus", "model": "IS", "year": 2006, "price": 22434, "created_at": "2021-01-16T10:45:52+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 50, "make": "Isuzu", "model": "VehiCROSS", "year": 2001, "price": 38180, "created_at": "2021-12-13T16:29:27+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 51, "make": "Buick", "model": "Regal", "year": 2000, "price": 38680, "created_at": "2021-12-29T22:25:54+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 52, "make": "Mercedes-Benz", "model": "E-Class", "year": 2007, "price": 51556, "created_at": "2021-07-06T11:42:23+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 53, "make": "Buick", "model": "LeSabre", "year": 2001, "price": 10904, "created_at": "2022-01-05T18:23:35+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 54, "make": "Porsche", "model": "928", "year": 1989, "price": 70917, "created_at": "2022-01-02T23:16:45+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 55, "make": "Lexus", "model": "RX", "year": 2007, "price": 5212, "created_at": "2021-07-10T15:02:53+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 56, "make": "Ford", "model": "Econoline E250", "year": 1996, "price": 75095, "created_at": "2021-02-04T16:17:18+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 57, "make": "Chevrolet", "model": "Blazer", "year": 2001, "price": 61918, "created_at": "2021-12-08T07:25:30+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 58, "make": "GMC", "model": "Savana 3500", "year": 2003, "price": 30307, "created_at": "2021-11-21T23:11:45+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 59, "make": "BMW", "model": "M", "year": 2002, "price": 24598, "created_at": "2021-05-28T04:08:53+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 60, "make": "Saturn", "model": "S-Series", "year": 1992, "price": 96288, "created_at": "2021-08-24T04:43:43+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 61, "make": "Chrysler", "model": "Sebring", "year": 2003, "price": 34753, "created_at": "2021-02-11T11:25:35+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 62, "make": "Lotus", "model": "Evora", "year": 2010, "price": 42760, "created_at": "2021-08-31T00:29:05+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 63, "make": "Jeep", "model": "Wrangler", "year": 2011, "price": 8684, "created_at": "2021-06-24T10:38:05+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 64, "make": "Ford", "model": "Expedition", "year": 2012, "price": 25653, "created_at": "2021-07-01T16:13:20+00:00"}, "emitted_at": 1671748915432} +{"stream": "products", "data": {"id": 65, "make": "Chevrolet", "model": "Avalanche 2500", "year": 2006, "price": 3158, "created_at": "2021-08-14T10:55:13+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 66, "make": "Mazda", "model": "Mazda3", "year": 2012, "price": 79820, "created_at": "2021-05-25T21:55:52+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 67, "make": "Toyota", "model": "Tacoma", "year": 2005, "price": 73572, "created_at": "2021-01-22T09:56:02+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 68, "make": "Ford", "model": "Explorer Sport", "year": 2000, "price": 64579, "created_at": "2021-02-16T06:56:06+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 69, "make": "GMC", "model": "Savana Cargo Van", "year": 2006, "price": 65944, "created_at": "2021-09-12T14:08:53+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 70, "make": "Chevrolet", "model": "HHR", "year": 2009, "price": 8953, "created_at": "2021-08-17T04:25:43+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 71, "make": "Ford", "model": "Bronco II", "year": 1989, "price": 41811, "created_at": "2021-07-14T14:20:28+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 72, "make": "Chevrolet", "model": "Suburban 2500", "year": 2011, "price": 57488, "created_at": "2021-09-22T12:32:57+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 73, "make": "Suzuki", "model": "Grand Vitara", "year": 2008, "price": 6408, "created_at": "2021-11-12T23:19:52+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 74, "make": "Mazda", "model": "Mazda6", "year": 2012, "price": 14805, "created_at": "2021-06-01T01:55:32+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 75, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 33585, "created_at": "2022-01-09T04:28:54+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 76, "make": "Ford", "model": "Explorer Sport Trac", "year": 2010, "price": 2087, "created_at": "2022-03-28T00:28:16+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 77, "make": "Ford", "model": "F150", "year": 2007, "price": 17621, "created_at": "2021-03-23T15:08:10+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 78, "make": "Ford", "model": "Taurus", "year": 1995, "price": 16478, "created_at": "2021-06-07T22:29:50+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 79, "make": "Mitsubishi", "model": "Truck", "year": 1992, "price": 70616, "created_at": "2022-01-30T05:14:02+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 80, "make": "Dodge", "model": "Colt", "year": 1994, "price": 34163, "created_at": "2022-04-02T18:06:30+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 81, "make": "Mazda", "model": "RX-7", "year": 1991, "price": 29634, "created_at": "2021-01-06T10:30:59+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 82, "make": "Pontiac", "model": "Grand Prix", "year": 1984, "price": 88575, "created_at": "2021-02-24T06:06:57+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 83, "make": "Mazda", "model": "Mazdaspeed 3", "year": 2012, "price": 77723, "created_at": "2021-11-11T22:48:05+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 84, "make": "Alfa Romeo", "model": "Spider", "year": 1992, "price": 64288, "created_at": "2021-01-06T03:50:27+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 85, "make": "Audi", "model": "S8", "year": 2002, "price": 33718, "created_at": "2021-07-21T11:14:54+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 86, "make": "Isuzu", "model": "Amigo", "year": 1992, "price": 53335, "created_at": "2022-03-02T10:42:21+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 87, "make": "Toyota", "model": "Paseo", "year": 1996, "price": 74558, "created_at": "2021-10-02 14:54:58+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 88, "make": "Lincoln", "model": "Continental Mark VII", "year": 1986, "price": 42150, "created_at": "2021-10-02T04:48:53+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 89, "make": "Dodge", "model": "Dakota", "year": 1997, "price": 64516, "created_at": "2021-09-09T23:13:26+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 90, "make": "Chevrolet", "model": "Tahoe", "year": 1998, "price": 51461, "created_at": "2021-04-06T08:29:19+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 91, "make": "Pontiac", "model": "Vibe", "year": 2006, "price": 12134, "created_at": "2021-01-11T22:30:14+00:00"}, "emitted_at": 1671748915433} +{"stream": "products", "data": {"id": 92, "make": "Volkswagen", "model": "Eos", "year": 2011, "price": 53128, "created_at": "2021-01-12T23:25:06+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 93, "make": "Mazda", "model": "Mazdaspeed6", "year": 2007, "price": 90902, "created_at": "2021-12-29T14:29:03+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 94, "make": "Nissan", "model": "Xterra", "year": 2005, "price": 41532, "created_at": "2021-09-07 09:00:49+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 95, "make": "Mercury", "model": "Sable", "year": 2005, "price": 71337, "created_at": "2021-01-31T22:13:44+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 96, "make": "BMW", "model": "330", "year": 2006, "price": 14494, "created_at": "2021-09-17T20:52:48+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 97, "make": "Audi", "model": "R8", "year": 2008, "price": 17642, "created_at": "2021-09-21T11:56:24+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 98, "make": "Cadillac", "model": "CTS-V", "year": 2007, "price": 19914, "created_at": "2021-09-02T15:38:46+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 99, "make": "GMC", "model": "1500 Club Coupe", "year": 1997, "price": 82288, "created_at": "2021-04-20T18:58:15+00:00"}, "emitted_at": 1671748915434} +{"stream": "products", "data": {"id": 100, "make": "Buick", "model": "Somerset", "year": 1986, "price": 64148, "created_at": "2021-06-10T19:07:38+00:00"}, "emitted_at": 1671748915434} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py new file mode 100644 index 000000000000..be338ddb4afe --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/airbyte_message_with_cached_json.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.models import AirbyteMessage + + +class AirbyteMessageWithCachedJSON(AirbyteMessage): + """ + I a monkeypatch to AirbyteMessage which pre-renders the JSON-representation of the object upon initialization. + This allows the JSON to be calculated in the process that builds the object rather than the main process. + + Note: We can't use @cache here because the LRU cache is not serializable when passed to child workers. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._json = self.json(exclude_unset=True) + self.json = self.get_json + + def get_json(self, **kwargs): + return self._json diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py new file mode 100644 index 000000000000..53cf05be4177 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from multiprocessing import current_process +from typing import Dict + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Numeric + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class PurchaseGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters + """ + + seed_with_offset = self.seed + if self.seed is not None and len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global dt + global numeric + + dt = Datetime(seed=seed_with_offset) + numeric = Numeric(seed=seed_with_offset) + + def random_date_in_range( + self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() + ) -> datetime.datetime: + time_between_dates = end_date - start_date + days_between_dates = time_between_dates.days + if days_between_dates < 2: + days_between_dates = 2 + random_number_of_days = numeric.integer_number(0, days_between_dates) + random_date = start_date + datetime.timedelta(days=random_number_of_days) + return random_date + + def generate(self, user_id: int) -> list[Dict]: + """ + Because we are doing this work in parallel processes, we need a deterministic way to know what a purchase's ID should be given on the input of a user_id. + tldr; Every 10 user_ids produce 10 purchases. User ID x5 has no purchases, User ID mod x7 has 2, and everyone else has 1 + """ + + purchases: list[Dict] = [] + last_user_id_digit = int(repr(user_id)[-1]) + purchase_count = 1 + id_offset = 0 + if last_user_id_digit - 1 == 5: + purchase_count = 0 + elif last_user_id_digit - 1 == 6: + id_offset = 1 + elif last_user_id_digit - 1 == 7: + id_offset = 1 + purchase_count = 2 + + total_products = 100 + i = 0 + + while purchase_count > 0: + id = user_id + i + 1 - id_offset + time_a = dt.datetime() + time_b = dt.datetime() + created_at = time_a if time_a <= time_b else time_b + product_id = numeric.integer_number(1, total_products) + added_to_cart_at = self.random_date_in_range(created_at) + purchased_at = ( + self.random_date_in_range(added_to_cart_at) + if added_to_cart_at is not None and numeric.integer_number(1, 100) <= 70 + else None + ) # 70% likely to purchase the item in the cart + returned_at = ( + self.random_date_in_range(purchased_at) if purchased_at is not None and numeric.integer_number(1, 100) <= 15 else None + ) # 15% likely to return the item + + purchase = { + "id": id, + "product_id": product_id, + "user_id": user_id + 1, + "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, + "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, + "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, + } + + record = AirbyteRecordMessage(stream=self.stream_name, data=purchase, emitted_at=now_millis()) + message = AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) + purchases.append(message) + + purchase_count = purchase_count - 1 + i += 1 + + return purchases diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index 25a35776e34e..3772cea75516 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -19,14 +19,14 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, "Count option is missing" def streams(self, config: Mapping[str, Any]) -> List[Stream]: - count: int = config["count"] if "count" in config else 0 seed: int = config["seed"] if "seed" in config else None records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500 records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100 + parallelism: int = config["parallelism"] if "parallelism" in config else 4 return [ - Products(count, seed, records_per_sync, records_per_slice), - Users(count, seed, records_per_sync, records_per_slice), - Purchases(count, seed, records_per_sync, records_per_slice), + Products(count, seed, parallelism, records_per_sync, records_per_slice), + Users(count, seed, parallelism, records_per_sync, records_per_slice), + Purchases(count, seed, parallelism, records_per_sync, records_per_slice), ] diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index 1018a957f109..0d20f791000d 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -35,8 +35,16 @@ "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", "type": "integer", "minimum": 1, - "default": 100, + "default": 1000, "order": 3 + }, + "parallelism": { + "title": "Parallelism", + "description": "How many parallel workers should we use to generate fake data? Choose a value equal to the number of CPUs you will allocate to this source.", + "type": "integer", + "minimum": 1, + "default": 4, + "order": 4 } } } diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index eab4969305f5..a25e93813615 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -2,23 +2,22 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import datetime import os +from multiprocessing import Pool from typing import Any, Dict, Iterable, Mapping, Optional -from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from mimesis import Datetime, Numeric, Person -from mimesis.locales import Locale -from .utils import format_airbyte_time, read_json +from .purchase_generator import PurchaseGenerator +from .user_generator import UserGenerator +from .utils import generate_estimate, read_json class Products(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.seed = seed self.records_per_sync = records_per_sync @@ -64,14 +63,14 @@ class Users(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.person = Person(locale=Locale.EN, seed=self.seed) - self.dt = Datetime(seed=self.seed) + self.parallelism = parallelism + self.generator = UserGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -88,58 +87,33 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def generate_user(self, user_id: int): - time_a = self.dt.datetime() - time_b = self.dt.datetime() - - profile = { - "id": user_id + 1, - "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), - "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), - "name": self.person.name(), - "title": self.person.title(), - "age": self.person.age(), - "email": self.person.email(), - "telephone": self.person.telephone(), - "gender": self.person.gender(), - "language": self.person.language(), - "academic_degree": self.person.academic_degree(), - "nationality": self.person.nationality(), - "occupation": self.person.occupation(), - "height": self.person.height(), - "blood_type": self.person.blood_type(), - "weight": self.person.weight(), - } - - while not profile["created_at"]: - profile["created_at"] = format_airbyte_time(self.dt.datetime()) - - if not profile["updated_at"]: - profile["updated_at"] = profile["created_at"] + 1 - - return profile - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ + total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 records_in_sync = 0 - records_in_slice = 0 median_record_byte_size = 450 yield generate_estimate(self.name, self.count - total_records, median_record_byte_size) - for i in range(total_records, self.count): - user = self.generate_user(i) - yield user - total_records += 1 - records_in_sync += 1 - records_in_slice += 1 + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while records_in_sync < self.count and records_in_sync < self.records_per_sync: + records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) + if records_remaining_this_loop <= 0: + break + users = pool.map(self.generator.generate, range(total_records, total_records + records_remaining_this_loop)) + for user in users: + total_records += 1 + records_in_sync += 1 + yield user - if records_in_slice >= self.records_per_slice: - self.state = {self.cursor_field: total_records, "seed": self.seed} - records_in_slice = 0 + if records_in_sync >= self.records_per_sync: + break - if records_in_sync == self.records_per_sync: - break + self.state = {self.cursor_field: total_records, "seed": self.seed} self.state = {self.cursor_field: total_records, "seed": self.seed} @@ -148,14 +122,14 @@ class Purchases(Stream, IncrementalMixin): primary_key = None cursor_field = "id" - def __init__(self, count: int, seed: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice - self.dt = Datetime(seed=self.seed) - self.numeric = Numeric(seed=self.seed) + self.parallelism = parallelism + self.generator = PurchaseGenerator(self.name, self.seed) @property def state_checkpoint_interval(self) -> Optional[int]: @@ -172,88 +146,37 @@ def state(self) -> Mapping[str, Any]: def state(self, value: Mapping[str, Any]): self._state = value - def random_date_in_range( - self, start_date: datetime.datetime, end_date: datetime.datetime = datetime.datetime.now() - ) -> datetime.datetime: - time_between_dates = end_date - start_date - days_between_dates = time_between_dates.days - if days_between_dates < 2: - days_between_dates = 2 - random_number_of_days = self.numeric.integer_number(0, days_between_dates) - random_date = start_date + datetime.timedelta(days=random_number_of_days) - return random_date - - def generate_purchases(self, user_id: int, purchases_count: int) -> list[Dict]: - purchases: list[Dict] = [] - purchase_percent_remaining = 70 # ~30% of people will have no purchases - total_products = 100 - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) - i = 0 - - time_a = self.dt.datetime() - time_b = self.dt.datetime() - created_at = time_a if time_a <= time_b else time_b - - while purchase_percent_remaining > 0: - id = purchases_count + i + 1 - product_id = self.numeric.integer_number(1, total_products) - added_to_cart_at = self.random_date_in_range(created_at) - purchased_at = ( - self.random_date_in_range(added_to_cart_at) - if added_to_cart_at is not None and self.numeric.integer_number(1, 100) <= 70 - else None - ) # 70% likely to purchase the item in the cart - returned_at = ( - self.random_date_in_range(purchased_at) if purchased_at is not None and self.numeric.integer_number(1, 100) <= 15 else None - ) # 15% likely to return the item - - purchase = { - "id": id, - "product_id": product_id, - "user_id": user_id, - "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, - "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, - "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, - } - purchases.append(purchase) - - purchase_percent_remaining = purchase_percent_remaining - self.numeric.integer_number(1, 100) - i += 1 - return purchases - def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This is a multi-process implementation of read_records. + We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON + """ + total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 total_user_records = self.state["user_id"] if "user_id" in self.state else 0 user_records_in_sync = 0 - user_records_in_slice = 0 + # a fuzzy guess, some users have purchases, some don't median_record_byte_size = 230 - yield generate_estimate( - self.name, (self.count - total_user_records) * 1.3, median_record_byte_size - ) # a fuzzy guess, some users have purchases, some don't - - for i in range(total_user_records, self.count): - purchases = self.generate_purchases(i + 1, total_purchase_records) - for purchase in purchases: - total_purchase_records += 1 - yield purchase - total_user_records += 1 - user_records_in_sync += 1 - user_records_in_slice += 1 - - if user_records_in_slice >= self.records_per_slice: - self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} - user_records_in_slice = 0 + yield generate_estimate(self.name, (self.count - total_user_records) * 1.3, median_record_byte_size) - if user_records_in_sync == self.records_per_sync: - break + with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: + while total_user_records < self.count and user_records_in_sync < self.records_per_sync: + records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) + if records_remaining_this_loop <= 0: + break + carts = pool.map(self.generator.generate, range(total_user_records, total_user_records + records_remaining_this_loop)) + for purchases in carts: + for purchase in purchases: + total_purchase_records += 1 + yield purchase - self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + total_user_records += 1 + user_records_in_sync += 1 + if user_records_in_sync >= self.records_per_sync: + break -def generate_estimate(stream_name: str, total: int, bytes_per_row: int): - emitted_at = int(datetime.datetime.now().timestamp() * 1000) - estimate_message = AirbyteEstimateTraceMessage( - type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) - ) - return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + + self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py new file mode 100644 index 000000000000..3fcedd2b6ad4 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from multiprocessing import current_process + +from airbyte_cdk.models import AirbyteRecordMessage, Type +from mimesis import Datetime, Person +from mimesis.locales import Locale + +from .airbyte_message_with_cached_json import AirbyteMessageWithCachedJSON +from .utils import format_airbyte_time, now_millis + + +class UserGenerator: + def __init__(self, stream_name: str, seed: int) -> None: + self.stream_name = stream_name + self.seed = seed + + def prepare(self): + """ + Note: the instances of the mimesis generators need to be global. + Yes, they *should* be able to be instance variables on this class, which should only instantiated once-per-worker, but that's not quite the case: + * relying only on prepare as a pool initializer fails because we are calling the parent process's method, not the fork + * Calling prepare() as part of generate() (perhaps checking if self.person is set) and then `print(self, current_process()._identity, current_process().pid)` reveals multiple object IDs in the same process, resetting the internal random counters + """ + + seed_with_offset = self.seed + if self.seed is not None and len(current_process()._identity) > 0: + seed_with_offset = self.seed + current_process()._identity[0] + + global person + global dt + + person = Person(locale=Locale.EN, seed=seed_with_offset) + dt = Datetime(seed=seed_with_offset) + + def generate(self, user_id: int): + time_a = dt.datetime() + time_b = dt.datetime() + + # faker doesn't always produce unique email addresses, so to enforce uniqueness, we will append the user_id to the prefix + email_parts = person.email().split("@") + email = f"{email_parts[0]}+{user_id + 1}@{email_parts[1]}" + + profile = { + "id": user_id + 1, + "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), + "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), + "name": person.name(), + "title": person.title(), + "age": person.age(), + "email": email, + "telephone": person.telephone(), + "gender": person.gender(), + "language": person.language(), + "academic_degree": person.academic_degree(), + "nationality": person.nationality(), + "occupation": person.occupation(), + "height": person.height(), + "blood_type": person.blood_type(), + "weight": person.weight(), + } + + while not profile["created_at"]: + profile["created_at"] = format_airbyte_time(dt.datetime()) + + if not profile["updated_at"]: + profile["updated_at"] = profile["created_at"] + 1 + + record = AirbyteRecordMessage(stream=self.stream_name, data=profile, emitted_at=now_millis()) + return AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/utils.py b/airbyte-integrations/connectors/source-faker/source_faker/utils.py index 12970853c956..9c094b11bcb7 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/utils.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/utils.py @@ -5,6 +5,8 @@ import datetime import json +from airbyte_cdk.models import AirbyteEstimateTraceMessage, AirbyteTraceMessage, EstimateType, TraceType + def read_json(filepath): with open(filepath, "r") as f: @@ -17,3 +19,15 @@ def format_airbyte_time(d: datetime): s = s.replace(" ", "T") s += "+00:00" return s + + +def now_millis(): + return int(datetime.datetime.now().timestamp() * 1000) + + +def generate_estimate(stream_name: str, total: int, bytes_per_row: int): + emitted_at = int(datetime.datetime.now().timestamp() * 1000) + estimate_message = AirbyteEstimateTraceMessage( + type=EstimateType.STREAM, name=stream_name, row_estimate=round(total), byte_estimate=round(total * bytes_per_row) + ) + return AirbyteTraceMessage(type=TraceType.ESTIMATE, emitted_at=emitted_at, estimate=estimate_message) diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index 8a989b7778ac..193960b73453 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -25,7 +25,7 @@ def exception(a,b,**kwargs): def schemas_are_valid(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -36,7 +36,7 @@ def schemas_are_valid(): def test_source_streams(): source = SourceFaker() - config = {"count": 1} + config = {"count": 1, "parallelism": 1} catalog = source.discover(None, config) catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] @@ -64,7 +64,7 @@ def test_source_streams(): def test_read_small_random_data(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -98,7 +98,7 @@ def test_read_small_random_data(): def test_no_read_limit_hit(): source = SourceFaker() - config = {"count": 10} + config = {"count": 10, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -128,7 +128,7 @@ def test_no_read_limit_hit(): def test_read_big_random_data(): source = SourceFaker() - config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000} + config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -163,7 +163,7 @@ def test_read_big_random_data(): def test_with_purchases(): source = SourceFaker() - config = {"count": 1000, "records_per_sync": 1000} + config = {"count": 1000, "records_per_sync": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -205,7 +205,7 @@ def test_with_purchases(): def test_sync_ends_with_limit(): source = SourceFaker() - config = {"count": 100, "records_per_sync": 5} + config = {"count": 100, "records_per_sync": 5, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -239,7 +239,7 @@ def test_read_with_seed(): """ source = SourceFaker() - config = {"count": 1, "seed": 100} + config = {"count": 1, "seed": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -253,14 +253,14 @@ def test_read_with_seed(): iterator = source.read(logger, config, catalog, state) records = [row for row in iterator if row.type is Type.RECORD] - assert records[0].record.data["occupation"] == "Roadworker" - assert records[0].record.data["email"] == "reproduce1856@outlook.com" + assert records[0].record.data["occupation"] == "Cartoonist" + assert records[0].record.data["email"] == "reflect1958+1@yahoo.com" def test_ensure_no_purchases_without_users(): with pytest.raises(ValueError): source = SourceFaker() - config = {"count": 100} + config = {"count": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ {"stream": {"name": "purchases", "json_schema": {}}, "sync_mode": "incremental", "destination_sync_mode": "overwrite"},