Add column level lineage endpoint proposal #2045
Comments
11 tasks
|
We limit the scope of this to merging proposal. |
pawel-big-lebowski
changed the title
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
OpenLineage has a column-level lineage facet.
Marquez could leverage this to implement a column-level lineage endpoint.
The detailed proposal can be found here: #2077
Context
Existing lineage endpoint:
returns all the reachable nodes up to a given number of hops in the graph:
Click to open the returned payload
{ "graph": [{ "id": "dataset:food_delivery:public.categories", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.categories" }, "type": "DB_TABLE", "name": "public.categories", "physicalName": "public.categories", "createdAt": "2021-03-09T02:33:18.468719Z", "updatedAt": "2022-08-04T05:08:09.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the category." }, { "name": "name", "type": "VARCHAR", "tags": [], "description": "The name of the category." }, { "name": "menu_id", "type": "INTEGER", "tags": [], "description": "The ID of the menu related to the category." }, { "name": "description", "type": "TEXT", "tags": [], "description": "The description of the category." }], "tags": [], "lastModifiedAt": "2022-08-04T05:03:09.190723Z", "description": null, "lastlifecycleState": null }, "inEdges": [{ "origin": "job:food_delivery:etl_orders.etl_categories", "destination": "dataset:food_delivery:public.categories" }], "outEdges": [{ "origin": "dataset:food_delivery:public.categories", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }] }, { "id": "dataset:food_delivery:public.customers", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.customers" }, "type": "DB_TABLE", "name": "public.customers", "physicalName": "public.customers", "createdAt": "2021-03-09T02:33:20.092476Z", "updatedAt": "2022-08-04T05:18:52.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the customer." }, { "name": "created_at", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the customer was created." }, { "name": "updated_at", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the customer was updated." }, { "name": "name", "type": "VARCHAR", "tags": [], "description": "The name of the customer." }, { "name": "email", "type": "VARCHAR", "tags": [], "description": "The email address of the customer." }, { "name": "address", "type": "VARCHAR", "tags": [], "description": "The address of the customer." }, { "name": "phone", "type": "VARCHAR", "tags": [], "description": "The phone number of the customer." }, { "name": "city_id", "type": "INTEGER", "tags": [], "description": "The ID of the city related to the customer." }], "tags": [], "lastModifiedAt": "2022-08-04T05:07:21.190723Z", "description": null, "lastlifecycleState": "" }, "inEdges": [{ "origin": "job:food_delivery:etl.etl_customers", "destination": "dataset:food_delivery:public.customers" }], "outEdges": [{ "origin": "dataset:food_delivery:public.customers", "destination": "job:food_delivery:analytics.orders_popular_day_of_week" }, { "origin": "dataset:food_delivery:public.customers", "destination": "job:food_delivery:emails.email_discounts" }, { "origin": "dataset:food_delivery:public.customers", "destination": "job:food_delivery:etl.etl_delivery_7_days" }] }, { "id": "dataset:food_delivery:public.delivery_7_days", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.delivery_7_days" }, "type": "DB_TABLE", "name": "public.delivery_7_days", "physicalName": "public.delivery_7_days", "createdAt": "2021-03-09T02:33:21.693799Z", "updatedAt": "2022-08-04T05:15:38.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "order_id", "type": "INTEGER", "tags": [], "description": "The ID of the order." }, { "name": "order_placed_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was placed." }, { "name": "order_dispatched_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was dispatched." }, { "name": "order_delivered_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was delivered." }, { "name": "customer_email", "type": "VARCHAR", "tags": [], "description": "The email of the customer." }, { "name": "menu_id", "type": "INTEGER", "tags": [], "description": "The ID of the menu related to the order." }, { "name": "menu_item_id", "type": "INTEGER", "tags": [], "description": "The ID of the menu item related to the order." }, { "name": "category_id", "type": "INTEGER", "tags": [], "description": "The ID of category related to the order." }, { "name": "discount_id", "type": "VARCHAR", "tags": [], "description": "The ID of the discount applied to the order." }, { "name": "city_id", "type": "INTEGER", "tags": [], "description": "The ID of the city related to the order." }, { "name": "driver_id", "type": "INTEGER", "tags": [], "description": "The ID of the driver related to the order." }], "tags": [], "lastModifiedAt": "2022-08-04T05:10:50.190723Z", "description": null, "lastlifecycleState": "" }, "inEdges": [{ "origin": "job:food_delivery:etl.etl_delivery_7_days", "destination": "dataset:food_delivery:public.delivery_7_days" }], "outEdges": [{ "origin": "dataset:food_delivery:public.delivery_7_days", "destination": "job:food_delivery:analytics.delivery_times_7_days" }] }, { "id": "dataset:food_delivery:public.discounts", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.discounts" }, "type": "DB_TABLE", "name": "public.discounts", "physicalName": "public.discounts", "createdAt": "2021-03-09T02:33:22.070729Z", "updatedAt": "2022-08-04T05:18:37.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the discount." }, { "name": "amount_off", "type": "INTEGER", "tags": [], "description": "The amount of the discount." }, { "name": "customer_email", "type": "VARCHAR", "tags": [], "description": "The email of the customer." }, { "name": "starts_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the discount starts." }, { "name": "ends_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the discount ends." }], "tags": [], "lastModifiedAt": "2022-08-04T05:15:38.190723Z", "description": null, "lastlifecycleState": null }, "inEdges": [{ "origin": "job:food_delivery:analytics.delivery_times_7_days", "destination": "dataset:food_delivery:public.discounts" }], "outEdges": [{ "origin": "dataset:food_delivery:public.discounts", "destination": "job:food_delivery:emails.email_discounts" }] }, { "id": "dataset:food_delivery:public.drivers", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.drivers" }, "type": "DB_TABLE", "name": "public.drivers", "physicalName": "public.drivers", "createdAt": "2021-03-09T02:33:21.253785Z", "updatedAt": "2022-08-04T05:10:50.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the driver." }, { "name": "created_at", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the driver was created." }, { "name": "updated_at", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the driver was updated." }, { "name": "name", "type": "VARCHAR", "tags": [], "description": "The name of the driver." }, { "name": "email", "type": "VARCHAR", "tags": [], "description": "The email of the driver." }, { "name": "phone", "type": "VARCHAR", "tags": [], "description": "The phone number of the driver." }, { "name": "car_make", "type": "VARCHAR", "tags": [], "description": "The make of the car." }, { "name": "car_model", "type": "VARCHAR", "tags": [], "description": "The model of the car." }, { "name": "car_year", "type": "VARCHAR", "tags": [], "description": "The year of the car." }, { "name": "car_color", "type": "VARCHAR", "tags": [], "description": "The color of the car." }, { "name": "car_license_plate", "type": "VARCHAR", "tags": [], "description": "The license plate number of the car." }], "tags": [], "lastModifiedAt": "2022-08-04T05:06:40.190723Z", "description": null, "lastlifecycleState": "" }, "inEdges": [{ "origin": "job:food_delivery:etl.etl_drivers", "destination": "dataset:food_delivery:public.drivers" }], "outEdges": [{ "origin": "dataset:food_delivery:public.drivers", "destination": "job:food_delivery:etl.etl_delivery_7_days" }] }, { "id": "dataset:food_delivery:public.menu_items", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.menu_items" }, "type": "DB_TABLE", "name": "public.menu_items", "physicalName": "public.menu_items", "createdAt": "2021-03-09T02:33:18.959288Z", "updatedAt": "2022-08-04T05:08:09.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the menu item." }, { "name": "name", "type": "VARCHAR", "tags": [], "description": "The name of the menu item." }, { "name": "price", "type": "INTEGER", "tags": [], "description": "The price of the menu item." }, { "name": "category_id", "type": "INTEGER", "tags": [], "description": "The ID of the category related to the item." }, { "name": "description", "type": "TEXT", "tags": [], "description": "The description of the menu item." }], "tags": [], "lastModifiedAt": "2022-08-04T05:03:24.190723Z", "description": null, "lastlifecycleState": null }, "inEdges": [{ "origin": "job:food_delivery:etl_orders.etl_menu_items", "destination": "dataset:food_delivery:public.menu_items" }], "outEdges": [{ "origin": "dataset:food_delivery:public.menu_items", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }] }, { "id": "dataset:food_delivery:public.menus", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.menus" }, "type": "DB_TABLE", "name": "public.menus", "physicalName": "public.menus", "createdAt": "2021-03-09T02:33:17.948768Z", "updatedAt": "2022-08-04T05:08:09.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the menu." }, { "name": "name", "type": "VARCHAR", "tags": [], "description": "The name of the menu." }, { "name": "restaurant_id", "type": "INTEGER", "tags": [], "description": "The ID of the restaurant related to the menu." }, { "name": "description", "type": "TEXT", "tags": [], "description": "The description of the menu." }], "tags": [], "lastModifiedAt": "2022-08-04T05:03:06.190723Z", "description": null, "lastlifecycleState": "" }, "inEdges": [{ "origin": "job:food_delivery:etl_orders.etl_menus", "destination": "dataset:food_delivery:public.menus" }], "outEdges": [{ "origin": "dataset:food_delivery:public.menus", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }] }, { "id": "dataset:food_delivery:public.order_status", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.order_status" }, "type": "DB_TABLE", "name": "public.order_status", "physicalName": "public.order_status", "createdAt": "2021-03-09T02:33:20.488093Z", "updatedAt": "2022-08-04T05:10:50.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the order status." }, { "name": "transitioned_at", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order status was transitioned." }, { "name": "status", "type": "VARCHAR", "tags": [], "description": "The status of the order status." }, { "name": "order_id", "type": "INTEGER", "tags": [], "description": "The ID of the order related to the order status." }, { "name": "customer_id", "type": "INTEGER", "tags": [], "description": "The ID of the customer related to the order status." }, { "name": "restaurant_id", "type": "INTEGER", "tags": [], "description": "The ID of the restaurant related to the order status." }, { "name": "driver_id", "type": "INTEGER", "tags": [], "description": "The ID of the driver related to the order status." }], "tags": [], "lastModifiedAt": "2022-08-04T05:07:05.190723Z", "description": null, "lastlifecycleState": null }, "inEdges": [{ "origin": "job:food_delivery:etl.etl_order_status", "destination": "dataset:food_delivery:public.order_status" }], "outEdges": [{ "origin": "dataset:food_delivery:public.order_status", "destination": "job:food_delivery:etl.etl_delivery_7_days" }] }, { "id": "dataset:food_delivery:public.orders", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.orders" }, "type": "DB_TABLE", "name": "public.orders", "physicalName": "public.orders", "createdAt": "2021-03-09T02:33:16.472042Z", "updatedAt": "2022-08-04T05:08:09.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the order." }, { "name": "placed_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was placed." }, { "name": "menu_item_id", "type": "INTEGER", "tags": [], "description": "The ID of the menu item related to the order." }, { "name": "quantity", "type": "INTEGER", "tags": [], "description": "The number of the item in the order." }, { "name": "discount_id", "type": "INTEGER", "tags": [], "description": "The unique ID of the discount applied to the order." }, { "name": "comment", "type": "VARCHAR", "tags": [], "description": "The comment of the order." }], "tags": [], "lastModifiedAt": "2022-08-04T05:03:01.190723Z", "description": null, "lastlifecycleState": null }, "inEdges": [{ "origin": "job:food_delivery:etl_orders.etl_orders", "destination": "dataset:food_delivery:public.orders" }], "outEdges": [{ "origin": "dataset:food_delivery:public.orders", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }] }, { "id": "dataset:food_delivery:public.orders_7_days", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.orders_7_days" }, "type": "DB_TABLE", "name": "public.orders_7_days", "physicalName": "public.orders_7_days", "createdAt": "2021-03-09T02:33:20.794369Z", "updatedAt": "2022-08-04T05:10:50.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "order_id", "type": "INTEGER", "tags": [], "description": "The ID of the order." }, { "name": "placed_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was placed." }, { "name": "menu_id", "type": "VARCHAR", "tags": [], "description": "The ID of the menu related to the order." }, { "name": "menu_item_id", "type": "INTEGER", "tags": [], "description": "The ID of the menu item related to the order." }, { "name": "category_id", "type": "INTEGER", "tags": [], "description": "The ID of category related to the order." }, { "name": "discount_id", "type": "INTEGER", "tags": [], "description": "The ID of the discount applied to the order." }, { "name": "city_id", "type": "INTEGER", "tags": [], "description": "The ID of the city related to the order." }], "tags": [], "lastModifiedAt": "2022-08-04T05:08:09.190723Z", "description": null, "lastlifecycleState": "" }, "inEdges": [{ "origin": "job:food_delivery:etl_orders.etl_orders_7_days", "destination": "dataset:food_delivery:public.orders_7_days" }], "outEdges": [{ "origin": "dataset:food_delivery:public.orders_7_days", "destination": "job:food_delivery:etl.etl_delivery_7_days" }] }, { "id": "dataset:food_delivery:public.popular_orders_day_of_week", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.popular_orders_day_of_week" }, "type": "DB_TABLE", "name": "public.popular_orders_day_of_week", "physicalName": "public.popular_orders_day_of_week", "createdAt": "2021-03-09T02:33:22.794238Z", "updatedAt": "2022-08-04T05:02:19.384484Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "order_day_of_week", "type": "VARCHAR", "tags": [], "description": "The day of week of the order." }, { "name": "order_placed_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was placed." }, { "name": "orders_placed", "type": "INTEGER", "tags": [], "description": "The number of orders placed on day of week." }], "tags": [], "lastModifiedAt": "2022-08-04T05:18:52.190723Z", "description": null, "lastlifecycleState": "" }, "inEdges": [{ "origin": "job:food_delivery:analytics.orders_popular_day_of_week", "destination": "dataset:food_delivery:public.popular_orders_day_of_week" }], "outEdges": [] }, { "id": "dataset:food_delivery:public.restaurants", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.restaurants" }, "type": "DB_TABLE", "name": "public.restaurants", "physicalName": "public.restaurants", "createdAt": "2021-03-09T02:33:19.475385Z", "updatedAt": "2022-08-04T05:10:50.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "id", "type": "INTEGER", "tags": [], "description": "The unique ID of the restaurant." }, { "name": "created_at", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the restaurant was created." }, { "name": "updated_at", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the restaurant was updated." }, { "name": "name", "type": "VARCHAR", "tags": [], "description": "The name of the restaurant." }, { "name": "email", "type": "VARCHAR", "tags": [], "description": "The email address of the customer." }, { "name": "address", "type": "VARCHAR", "tags": [], "description": "The address of the restaurant." }, { "name": "phone", "type": "VARCHAR", "tags": [], "description": "The phone number of the restaurant." }, { "name": "city_id", "type": "INTEGER", "tags": [], "description": "The ID of the city related to the restaurant." }, { "name": "business_hours_id", "type": "INTEGER", "tags": [], "description": "The ID of the business hours related to the restaurant." }, { "name": "description", "type": "TEXT", "tags": [], "description": "The description of the restaurant." }], "tags": [], "lastModifiedAt": "2022-08-04T05:06:12.190723Z", "description": null, "lastlifecycleState": "" }, "inEdges": [{ "origin": "job:food_delivery:etl.etl_restaurants", "destination": "dataset:food_delivery:public.restaurants" }], "outEdges": [{ "origin": "dataset:food_delivery:public.restaurants", "destination": "job:food_delivery:etl.etl_delivery_7_days" }] }, { "id": "dataset:food_delivery:public.top_delivery_times", "type": "DATASET", "data": { "type": "DATASET", "id": { "namespace": "food_delivery", "name": "public.top_delivery_times" }, "type": "DB_TABLE", "name": "public.top_delivery_times", "physicalName": "public.top_delivery_times", "createdAt": "2021-03-09T02:33:22.352665Z", "updatedAt": "2022-08-04T05:18:52.190723Z", "namespace": "food_delivery", "sourceName": "analytics_db", "fields": [{ "name": "order_id", "type": "INTEGER", "tags": [], "description": "The ID of the order." }, { "name": "order_placed_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was placed." }, { "name": "order_dispatched_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was dispatched." }, { "name": "order_delivered_on", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the date/time the order was delivered." }, { "name": "order_delivered_time", "type": "TIMESTAMP", "tags": [], "description": "An ISO-8601 timestamp representing the total time of delivery." }, { "name": "customer_email", "type": "VARCHAR", "tags": [], "description": "The email of the customer." }, { "name": "restaurant_id", "type": "INTEGER", "tags": [], "description": "The ID of the restaurant related to the order." }, { "name": "driver_id", "type": "INTEGER", "tags": [], "description": "The ID of the driver related to the order." }], "tags": [], "lastModifiedAt": "2022-08-04T05:15:38.190723Z", "description": null, "lastlifecycleState": null }, "inEdges": [{ "origin": "job:food_delivery:analytics.delivery_times_7_days", "destination": "dataset:food_delivery:public.top_delivery_times" }], "outEdges": [{ "origin": "dataset:food_delivery:public.top_delivery_times", "destination": "job:food_delivery:analytics.orders_popular_day_of_week" }] }, { "id": "job:food_delivery:analytics.delivery_times_7_days", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "analytics.delivery_times_7_days" }, "type": "BATCH", "name": "analytics.delivery_times_7_days", "createdAt": "2021-07-21T05:00:12.458719Z", "updatedAt": "2022-08-04T05:15:38.190723Z", "namespace": "food_delivery", "inputs": [{ "namespace": "food_delivery", "name": "public.delivery_7_days" }], "outputs": [{ "namespace": "food_delivery", "name": "public.top_delivery_times" }, { "namespace": "food_delivery", "name": "public.discounts" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/delivery_times_7_days.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/delivery_times_7_days.py", "sql": "INSERT INTO top_delivery_times (order_id, order_placed_on, order_dispatched_on, order_delivered_on, order_delivery_time,\n customer_email, restaurant_id, driver_id)\n SELECT order_id, order_placed_on, order_delivered_on, DATEDIFF(minute, order_placed_on, order_delivered_on) AS order_delivery_time,\n customer_email, restaurant_id, driver_id\n FROM delivery_7_days\nGROUP BY restaurant_id\nORDER BY order_delivery_time DESC\n LIMIT 1;" }, "description": "Determine weekly top delivery times by restaurant.", "latestRun": { "id": "3d4d4573-6419-47f7-ae0f-e0ed17784beb", "createdAt": "2022-08-04T05:02:18.275431Z", "updatedAt": "2022-08-04T05:15:38.190723Z", "nominalStartTime": "2022-08-04T05:12:00Z", "nominalEndTime": "2022-08-04T05:15:00Z", "state": "FAILED", "startedAt": "2022-08-04T05:12:12.190723Z", "endedAt": "2022-08-04T05:15:38.190723Z", "durationMs": 206000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "analytics.delivery_times_7_days", "version": "41945ccd-36d0-3eb4-b3dd-a78fdf78aa79" }, "inputVersions": [{ "namespace": "food_delivery", "name": "public.delivery_7_days", "version": "3129ce46-2ef4-3cce-8f47-6ffdee72f7a9" }], "outputVersions": [{ "namespace": "food_delivery", "name": "public.top_delivery_times", "version": "c8fc976a-2eda-353d-8e87-0f2be6b6b5ba" }, { "namespace": "food_delivery", "name": "public.discounts", "version": "0f480ffa-a812-33d1-8783-7181a82de578" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/delivery_times_7_days.py", "sql": "INSERT INTO top_delivery_times (order_id, order_placed_on, order_dispatched_on, order_delivered_on, order_delivery_time,\n customer_email, restaurant_id, driver_id)\n SELECT order_id, order_placed_on, order_delivered_on, DATEDIFF(minute, order_placed_on, order_delivered_on) AS order_delivery_time,\n customer_email, restaurant_id, driver_id\n FROM delivery_7_days\nGROUP BY restaurant_id\nORDER BY order_delivery_time DESC\n LIMIT 1;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:15:00Z", "nominalStartTime": "2022-08-04T05:12:00Z" } } } }, "inEdges": [{ "origin": "dataset:food_delivery:public.delivery_7_days", "destination": "job:food_delivery:analytics.delivery_times_7_days" }], "outEdges": [{ "origin": "job:food_delivery:analytics.delivery_times_7_days", "destination": "dataset:food_delivery:public.discounts" }, { "origin": "job:food_delivery:analytics.delivery_times_7_days", "destination": "dataset:food_delivery:public.top_delivery_times" }] }, { "id": "job:food_delivery:analytics.orders_popular_day_of_week", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "analytics.orders_popular_day_of_week" }, "type": "BATCH", "name": "analytics.orders_popular_day_of_week", "createdAt": "2021-07-21T05:00:12.665794Z", "updatedAt": "2022-08-04T05:18:52.190723Z", "namespace": "food_delivery", "inputs": [{ "namespace": "food_delivery", "name": "public.top_delivery_times" }, { "namespace": "food_delivery", "name": "public.customers" }], "outputs": [{ "namespace": "food_delivery", "name": "public.popular_orders_day_of_week" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/orders_popular_day_of_week.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/orders_popular_day_of_week.py", "sql": "INSERT INTO popular_orders_day_of_week (order_day_of_week, order_placed_on, orders_placed)\n SELECT order_day_of_week, order_placed_on, COUNT(*)\n FROM top_delivery_times;" }, "description": "Determines the popular day of week orders are placed.", "latestRun": { "id": "f9eb940e-01b6-408c-bc8d-e73cea1531b7", "createdAt": "2022-08-04T05:02:18.984244Z", "updatedAt": "2022-08-04T05:18:52.190723Z", "nominalStartTime": "2022-08-04T05:16:00Z", "nominalEndTime": "2022-08-04T05:18:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:16:12.190723Z", "endedAt": "2022-08-04T05:18:52.190723Z", "durationMs": 160000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "analytics.orders_popular_day_of_week", "version": "a9ab4908-971d-3989-b2ae-1f3c61a52f9e" }, "inputVersions": [{ "namespace": "food_delivery", "name": "public.customers", "version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82" }, { "namespace": "food_delivery", "name": "public.top_delivery_times", "version": "16582b10-76c4-3821-baaf-c3c724f5fb52" }], "outputVersions": [{ "namespace": "food_delivery", "name": "public.popular_orders_day_of_week", "version": "72b355d1-3654-389c-b63c-2d2ac508a49a" }, { "namespace": "food_delivery", "name": "public.popular_orders_day_of_week", "version": "2f87c49c-3aee-3414-9645-09ddd13e2635" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/orders_popular_day_of_week.py", "sql": "INSERT INTO popular_orders_day_of_week (order_day_of_week, order_placed_on, orders_placed)\n SELECT order_day_of_week, order_placed_on, COUNT(*)\n FROM top_delivery_times;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:18:00Z", "nominalStartTime": "2022-08-04T05:16:00Z" } } } }, "inEdges": [{ "origin": "dataset:food_delivery:public.customers", "destination": "job:food_delivery:analytics.orders_popular_day_of_week" }, { "origin": "dataset:food_delivery:public.top_delivery_times", "destination": "job:food_delivery:analytics.orders_popular_day_of_week" }], "outEdges": [{ "origin": "job:food_delivery:analytics.orders_popular_day_of_week", "destination": "dataset:food_delivery:public.popular_orders_day_of_week" }] }, { "id": "job:food_delivery:emails.email_discounts", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "emails.email_discounts" }, "type": "BATCH", "name": "emails.email_discounts", "createdAt": "2021-07-21T05:00:12.563025Z", "updatedAt": "2022-08-04T05:18:37.190723Z", "namespace": "food_delivery", "inputs": [{ "namespace": "food_delivery", "name": "public.customers" }, { "namespace": "food_delivery", "name": "public.discounts" }], "outputs": [], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/email_discounts.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/email_discounts.py", "sql": "SELECT * FROM discounts;" }, "description": "Email discounts to customers that have experienced order delays.", "latestRun": { "id": "768ec627-2c32-40d2-b42e-1a42ada49221", "createdAt": "2022-08-04T05:02:18.671344Z", "updatedAt": "2022-08-04T05:18:37.190723Z", "nominalStartTime": "2022-08-04T05:16:00Z", "nominalEndTime": "2022-08-04T05:18:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:16:12.190723Z", "endedAt": "2022-08-04T05:18:37.190723Z", "durationMs": 145000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "emails.email_discounts", "version": "84c93939-abfa-353f-9bd6-3fded9ae2e84" }, "inputVersions": [{ "namespace": "food_delivery", "name": "public.discounts", "version": "93c4e393-0ed3-3aba-bd2b-787f5fcf9077" }, { "namespace": "food_delivery", "name": "public.customers", "version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82" }], "outputVersions": [], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/email_discounts.py", "sql": "SELECT * FROM discounts;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:18:00Z", "nominalStartTime": "2022-08-04T05:16:00Z" } } } }, "inEdges": [{ "origin": "dataset:food_delivery:public.customers", "destination": "job:food_delivery:emails.email_discounts" }, { "origin": "dataset:food_delivery:public.discounts", "destination": "job:food_delivery:emails.email_discounts" }], "outEdges": [] }, { "id": "job:food_delivery:etl.etl_customers", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl.etl_customers" }, "type": "BATCH", "name": "etl.etl_customers", "createdAt": "2021-07-21T05:00:12.074910Z", "updatedAt": "2022-08-04T05:07:21.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.customers" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_customers.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_customers.py", "sql": "INSERT INTO customers (id, created_at, updated_at, name, email, phone, city_id)\n SELECT id, created_at, updated_at, name, email, phone, city_id\n FROM tmp_customers;" }, "description": "Loads newly registered customers daily.", "latestRun": { "id": "c06999f5-4fe0-443a-bf52-37c329019fc6", "createdAt": "2022-08-04T05:02:15.705905Z", "updatedAt": "2022-08-04T05:07:21.190723Z", "nominalStartTime": "2022-08-04T05:04:00Z", "nominalEndTime": "2022-08-04T05:07:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:04:12.190723Z", "endedAt": "2022-08-04T05:07:21.190723Z", "durationMs": 189000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl.etl_customers", "version": "2fcacd14-f730-358c-b5ba-9bd3055f606b" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.customers", "version": "be2c578f-d09a-393f-b7f2-a6c26f82d046" }, { "namespace": "food_delivery", "name": "public.customers", "version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_customers.py", "sql": "INSERT INTO customers (id, created_at, updated_at, name, email, phone, city_id)\n SELECT id, created_at, updated_at, name, email, phone, city_id\n FROM tmp_customers;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:07:00Z", "nominalStartTime": "2022-08-04T05:04:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl.etl_customers", "destination": "dataset:food_delivery:public.customers" }] }, { "id": "job:food_delivery:etl.etl_delivery_7_days", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl.etl_delivery_7_days" }, "type": "BATCH", "name": "etl.etl_delivery_7_days", "createdAt": "2021-07-21T05:00:12.360090Z", "updatedAt": "2022-08-04T05:10:50.190723Z", "namespace": "food_delivery", "inputs": [{ "namespace": "food_delivery", "name": "public.restaurants" }, { "namespace": "food_delivery", "name": "public.customers" }, { "namespace": "food_delivery", "name": "public.drivers" }, { "namespace": "food_delivery", "name": "public.order_status" }, { "namespace": "food_delivery", "name": "public.orders_7_days" }], "outputs": [{ "namespace": "food_delivery", "name": "public.delivery_7_days" }], "location": "https://github.com/DatakinHQ/demo/blob/16adfec8fd1f37ad58e0d3c0102b3e00c4f02837/airflow/dags/etl_delivery_7_days.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/16adfec8fd1f37ad58e0d3c0102b3e00c4f02837/airflow/dags/etl_delivery_7_days.py", "sql": "INSERT INTO delivery (order_id, order_placed_on, order_dispatched_on, order_delivered_on, customer_email,\n customer_address, discount_id, menu_id, restaurant_id, restaurant_address, menu_item_id, category_id, driver_id)\n SELECT o.order_id, o.placed_on AS order_placed_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DISPATCHED') AS order_dispatched_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DELIVERED') AS order_delivered_on,\n c.email AS customer_email, c.address AS customer_address, o.discount_id, o.menu_id, o.restaurant_id,\n r.address, o.menu_item_id, o.category_id, d.id AS driver_id\n FROM orders_7_days AS o\n INNER JOIN order_status AS os\n ON os.order_id = o.order_id\n INNER JOIN customers AS c\n ON c.id = os.customer_id\n INNER JOIN restaurants AS r\n ON r.id = os.restaurant_id\n INNER JOIN drivers AS d\n ON d.id = os.driver_id\n WHERE os.transitioned_at >= NOW() - interval '7 days';" }, "description": "Loads new deliveries for the week.", "latestRun": { "id": "cda33f0a-5ee1-4be7-aa77-e553b9ff8fdc", "createdAt": "2022-08-04T05:02:17.321081Z", "updatedAt": "2022-08-04T05:10:50.190723Z", "nominalStartTime": "2022-08-04T05:08:00Z", "nominalEndTime": "2022-08-04T05:10:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:08:12.190723Z", "endedAt": "2022-08-04T05:10:50.190723Z", "durationMs": 158000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl.etl_delivery_7_days", "version": "bc6c294b-b0eb-3160-a06d-1ff9ba3a4e1c" }, "inputVersions": [{ "namespace": "food_delivery", "name": "public.orders_7_days", "version": "3dc3a809-8bdc-3c50-809d-fa24e5d48455" }, { "namespace": "food_delivery", "name": "public.order_status", "version": "1261ea1c-cb68-3ad1-b208-6c4c65714b8e" }, { "namespace": "food_delivery", "name": "public.drivers", "version": "9db6f800-ffcc-3eac-9c7a-86cd9d702c27" }, { "namespace": "food_delivery", "name": "public.restaurants", "version": "fcb2cece-ccd6-3fe0-a1c7-52bd0e320040" }, { "namespace": "food_delivery", "name": "public.customers", "version": "7f93c4de-e30c-39ad-84e6-a8b812c51d82" }], "outputVersions": [{ "namespace": "food_delivery", "name": "public.delivery_7_days", "version": "66fcf19a-e24a-3520-9d24-d78d75572066" }, { "namespace": "food_delivery", "name": "public.delivery_7_days", "version": "3129ce46-2ef4-3cce-8f47-6ffdee72f7a9" }], "context": { "sql": "INSERT INTO delivery (order_id, order_placed_on, order_dispatched_on, order_delivered_on, customer_email,\n customer_address, discount_id, menu_id, restaurant_id, restaurant_address, menu_item_id, category_id, driver_id)\n SELECT o.order_id, o.placed_on AS order_placed_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DISPATCHED') AS order_dispatched_on,\n (SELECT transitioned_at FROM order_status WHERE order_id == o.order_id AND status = 'DELIVERED') AS order_delivered_on,\n c.email AS customer_email, c.address AS customer_address, o.discount_id, o.menu_id, o.restaurant_id,\n r.address, o.menu_item_id, o.category_id, d.id AS driver_id\n FROM orders_7_days AS o\n INNER JOIN order_status AS os\n ON os.order_id = o.order_id\n INNER JOIN customers AS c\n ON c.id = os.customer_id\n INNER JOIN restaurants AS r\n ON r.id = os.restaurant_id\n INNER JOIN drivers AS d\n ON d.id = os.driver_id\n WHERE os.transitioned_at >= NOW() - interval '7 days';" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:10:00Z", "nominalStartTime": "2022-08-04T05:08:00Z" } } } }, "inEdges": [{ "origin": "dataset:food_delivery:public.customers", "destination": "job:food_delivery:etl.etl_delivery_7_days" }, { "origin": "dataset:food_delivery:public.drivers", "destination": "job:food_delivery:etl.etl_delivery_7_days" }, { "origin": "dataset:food_delivery:public.order_status", "destination": "job:food_delivery:etl.etl_delivery_7_days" }, { "origin": "dataset:food_delivery:public.orders_7_days", "destination": "job:food_delivery:etl.etl_delivery_7_days" }, { "origin": "dataset:food_delivery:public.restaurants", "destination": "job:food_delivery:etl.etl_delivery_7_days" }], "outEdges": [{ "origin": "job:food_delivery:etl.etl_delivery_7_days", "destination": "dataset:food_delivery:public.delivery_7_days" }] }, { "id": "job:food_delivery:etl.etl_drivers", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl.etl_drivers" }, "type": "BATCH", "name": "etl.etl_drivers", "createdAt": "2021-07-21T05:00:12.175032Z", "updatedAt": "2022-08-04T05:06:40.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.drivers" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_drivers.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_drivers.py", "sql": "INSERT INTO drivers (id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate)\n SELECT id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate\n FROM tmp_drivers;" }, "description": "Loads newly registered drivers daily.", "latestRun": { "id": "af9b88b0-af2e-44b0-a076-a5b21f54faba", "createdAt": "2022-08-04T05:02:16.321336Z", "updatedAt": "2022-08-04T05:06:40.190723Z", "nominalStartTime": "2022-08-04T05:04:00Z", "nominalEndTime": "2022-08-04T05:06:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:04:12.190723Z", "endedAt": "2022-08-04T05:06:40.190723Z", "durationMs": 148000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl.etl_drivers", "version": "c552ff83-87a2-386f-8c5d-c62c66773c06" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.drivers", "version": "7fbda958-e49f-3852-80a4-1878b8d29aeb" }, { "namespace": "food_delivery", "name": "public.drivers", "version": "9db6f800-ffcc-3eac-9c7a-86cd9d702c27" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_drivers.py", "sql": "INSERT INTO drivers (id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate)\n SELECT id, created_at, updated_at, name, email, phone, car_make, car_model, car_year, car_color, car_license_plate\n FROM tmp_drivers;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:06:00Z", "nominalStartTime": "2022-08-04T05:04:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl.etl_drivers", "destination": "dataset:food_delivery:public.drivers" }] }, { "id": "job:food_delivery:etl.etl_order_status", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl.etl_order_status" }, "type": "BATCH", "name": "etl.etl_order_status", "createdAt": "2021-07-21T05:00:12.157147Z", "updatedAt": "2022-08-04T05:07:05.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.order_status" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_order_status.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_order_status.py", "sql": "INSERT INTO order_status (id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id)\n SELECT id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id\n FROM tmp_order_status;" }, "description": "Loads order statues updates daily.", "latestRun": { "id": "a847b17b-7772-4606-b0d3-f0371a4358eb", "createdAt": "2022-08-04T05:02:16.018871Z", "updatedAt": "2022-08-04T05:07:05.190723Z", "nominalStartTime": "2022-08-04T05:04:00Z", "nominalEndTime": "2022-08-04T05:07:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:04:12.190723Z", "endedAt": "2022-08-04T05:07:05.190723Z", "durationMs": 173000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl.etl_order_status", "version": "2e04b236-747d-3ba2-a982-1acadda139d1" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.order_status", "version": "2298c453-1a77-3daf-baa7-77057a89d1c1" }, { "namespace": "food_delivery", "name": "public.order_status", "version": "1261ea1c-cb68-3ad1-b208-6c4c65714b8e" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_order_status.py", "sql": "INSERT INTO order_status (id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id)\n SELECT id, transitioned_at, status, order_id, customer_id, restaurant_id, driver_id\n FROM tmp_order_status;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:07:00Z", "nominalStartTime": "2022-08-04T05:04:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl.etl_order_status", "destination": "dataset:food_delivery:public.order_status" }] }, { "id": "job:food_delivery:etl.etl_restaurants", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl.etl_restaurants" }, "type": "BATCH", "name": "etl.etl_restaurants", "createdAt": "2021-07-21T05:00:12.052261Z", "updatedAt": "2022-08-04T05:06:12.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.restaurants" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_restaurants.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_restaurants.py", "sql": "INSERT INTO restaurants (id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description)\n SELECT id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description\n FROM tmp_restaurants;" }, "description": "Loads newly registered restaurants daily.", "latestRun": { "id": "e36d9152-9c01-4901-ad51-0d54e176ab00", "createdAt": "2022-08-04T05:02:15.375517Z", "updatedAt": "2022-08-04T05:06:12.190723Z", "nominalStartTime": "2022-08-04T05:04:00Z", "nominalEndTime": "2022-08-04T05:06:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:04:12.190723Z", "endedAt": "2022-08-04T05:06:12.190723Z", "durationMs": 120000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl.etl_restaurants", "version": "d51962aa-cf5f-3e0e-b240-fc1e5dbf4879" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.restaurants", "version": "3ba477ce-4016-3852-8d38-25bf68cdc1cd" }, { "namespace": "food_delivery", "name": "public.restaurants", "version": "fcb2cece-ccd6-3fe0-a1c7-52bd0e320040" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_restaurants.py", "sql": "INSERT INTO restaurants (id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description)\n SELECT id, created_at, updated_at, name, email, address, phone, city_id, business_hours_id, description\n FROM tmp_restaurants;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:06:00Z", "nominalStartTime": "2022-08-04T05:04:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl.etl_restaurants", "destination": "dataset:food_delivery:public.restaurants" }] }, { "id": "job:food_delivery:etl_orders.etl_categories", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl_orders.etl_categories" }, "type": "BATCH", "name": "etl_orders.etl_categories", "createdAt": "2021-07-21T05:00:11.870558Z", "updatedAt": "2022-08-04T05:03:09.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.categories" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_categories.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_categories.py", "sql": "INSERT INTO categories (id, name, menu_id, description)\n SELECT id, name, menu_id, description\n FROM tmp_categories;" }, "description": "Loads newly added menus categories daily.", "latestRun": { "id": "24a62ce4-4239-4aa4-b1c2-d1e3950349cc", "createdAt": "2022-08-04T05:02:14.592636Z", "updatedAt": "2022-08-04T05:03:09.190723Z", "nominalStartTime": "2022-08-04T05:00:00Z", "nominalEndTime": "2022-08-04T05:03:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:00:12.190723Z", "endedAt": "2022-08-04T05:03:09.190723Z", "durationMs": 177000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl_orders.etl_categories", "version": "4c409e02-f046-34e0-9336-a3f71c2af478" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.categories", "version": "aba09e19-b23e-3601-987d-590965c6aae3" }, { "namespace": "food_delivery", "name": "public.categories", "version": "d4ea4286-d8f3-3640-bad0-29afa61eb52d" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_categories.py", "sql": "INSERT INTO categories (id, name, menu_id, description)\n SELECT id, name, menu_id, description\n FROM tmp_categories;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:03:00Z", "nominalStartTime": "2022-08-04T05:00:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl_orders.etl_categories", "destination": "dataset:food_delivery:public.categories" }] }, { "id": "job:food_delivery:etl_orders.etl_menu_items", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl_orders.etl_menu_items" }, "type": "BATCH", "name": "etl_orders.etl_menu_items", "createdAt": "2021-07-21T05:00:11.962073Z", "updatedAt": "2022-08-04T05:03:24.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.menu_items" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menu_items.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menu_items.py", "sql": "INSERT INTO menu_items (id, name, price, category_id, description)\n SELECT id, name, price, category_id, description\n FROM tmp_menu_items;" }, "description": "Loads newly added restaurant menu items daily.", "latestRun": { "id": "6ca9eafe-ebbd-4edb-803e-f4c6772528db", "createdAt": "2022-08-04T05:02:15.086486Z", "updatedAt": "2022-08-04T05:03:24.190723Z", "nominalStartTime": "2022-08-04T05:00:00Z", "nominalEndTime": "2022-08-04T05:03:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:00:12.190723Z", "endedAt": "2022-08-04T05:03:24.190723Z", "durationMs": 192000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl_orders.etl_menu_items", "version": "b7bbdfa9-32e0-39c5-93d1-83a4d406d4b6" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.menu_items", "version": "80a5d913-ec71-3aeb-bb61-a14763dad31d" }, { "namespace": "food_delivery", "name": "public.menu_items", "version": "ee00a374-4a87-3606-8447-b4000aa5090b" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menu_items.py", "sql": "INSERT INTO menu_items (id, name, price, category_id, description)\n SELECT id, name, price, category_id, description\n FROM tmp_menu_items;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:03:00Z", "nominalStartTime": "2022-08-04T05:00:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl_orders.etl_menu_items", "destination": "dataset:food_delivery:public.menu_items" }] }, { "id": "job:food_delivery:etl_orders.etl_menus", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl_orders.etl_menus" }, "type": "BATCH", "name": "etl_orders.etl_menus", "createdAt": "2021-07-21T05:00:11.849068Z", "updatedAt": "2022-08-04T05:03:06.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.menus" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menus.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menus.py", "sql": "INSERT INTO menus (id, name, restaurant_id, description)\n SELECT id, name, restaurant_id, description\n FROM tmp_menus;" }, "description": "Loads newly added restaurant menus daily.", "latestRun": { "id": "418fe5ef-3205-42c9-bfc6-11d4c89f71d0", "createdAt": "2022-08-04T05:02:14.310639Z", "updatedAt": "2022-08-04T05:03:06.190723Z", "nominalStartTime": "2022-08-04T05:00:00Z", "nominalEndTime": "2022-08-04T05:03:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:00:12.190723Z", "endedAt": "2022-08-04T05:03:06.190723Z", "durationMs": 174000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl_orders.etl_menus", "version": "f71aea64-be54-30b9-a370-e43fa6c5d518" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.menus", "version": "a25c18e1-737a-3121-b9d8-e50fbd5ccf4a" }, { "namespace": "food_delivery", "name": "public.menus", "version": "5b4890a4-863c-3a87-8f88-99d64f0d38dc" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_menus.py", "sql": "INSERT INTO menus (id, name, restaurant_id, description)\n SELECT id, name, restaurant_id, description\n FROM tmp_menus;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:03:00Z", "nominalStartTime": "2022-08-04T05:00:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl_orders.etl_menus", "destination": "dataset:food_delivery:public.menus" }] }, { "id": "job:food_delivery:etl_orders.etl_orders", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl_orders.etl_orders" }, "type": "BATCH", "name": "etl_orders.etl_orders", "createdAt": "2021-07-21T05:00:11.754754Z", "updatedAt": "2022-08-04T05:03:01.190723Z", "namespace": "food_delivery", "inputs": [], "outputs": [{ "namespace": "food_delivery", "name": "public.orders" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders.py", "sql": "INSERT INTO orders (id, placed_on, menu_item_id, quantity, discount_id, comment)\n SELECT id, placed_on, menu_item_id, quantity, discount_id, comment\n FROM tmp_orders;" }, "description": "Loads newly placed orders weekly.", "latestRun": { "id": "9c498876-50e0-4767-b57c-7eb898f48bd2", "createdAt": "2022-08-04T05:02:14.012477Z", "updatedAt": "2022-08-04T05:03:01.190723Z", "nominalStartTime": "2022-08-04T05:00:00Z", "nominalEndTime": "2022-08-04T05:03:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:00:12.190723Z", "endedAt": "2022-08-04T05:03:01.190723Z", "durationMs": 169000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl_orders.etl_orders", "version": "5cd12692-13b8-3409-a80a-d670c77ba9af" }, "inputVersions": [], "outputVersions": [{ "namespace": "food_delivery", "name": "public.orders", "version": "acbe5e24-fcfc-3323-abed-60e12f6016f4" }, { "namespace": "food_delivery", "name": "public.orders", "version": "42d5a311-e3df-3ab6-bc8b-baed3a5910b0" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders.py", "sql": "INSERT INTO orders (id, placed_on, menu_item_id, quantity, discount_id, comment)\n SELECT id, placed_on, menu_item_id, quantity, discount_id, comment\n FROM tmp_orders;" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:03:00Z", "nominalStartTime": "2022-08-04T05:00:00Z" } } } }, "inEdges": [], "outEdges": [{ "origin": "job:food_delivery:etl_orders.etl_orders", "destination": "dataset:food_delivery:public.orders" }] }, { "id": "job:food_delivery:etl_orders.etl_orders_7_days", "type": "JOB", "data": { "type": "JOB", "id": { "namespace": "food_delivery", "name": "etl_orders.etl_orders_7_days" }, "type": "BATCH", "name": "etl_orders.etl_orders_7_days", "createdAt": "2021-07-21T05:00:12.264069Z", "updatedAt": "2022-08-04T05:08:09.190723Z", "namespace": "food_delivery", "inputs": [{ "namespace": "food_delivery", "name": "public.categories" }, { "namespace": "food_delivery", "name": "public.menu_items" }, { "namespace": "food_delivery", "name": "public.orders" }, { "namespace": "food_delivery", "name": "public.menus" }], "outputs": [{ "namespace": "food_delivery", "name": "public.orders_7_days" }], "location": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders_7_days.py", "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders_7_days.py", "sql": "INSERT INTO orders_7_days (order_id, placed_on, discount_id, menu_id, restaurant_id, menu_item_id, category_id)\n SELECT o.id AS order_id, o.placed_on, o.discount_id, m.id AS menu_id, m.restaurant_id, mi.id AS menu_item_id, c.id AS category_id\n FROM orders AS o\n INNER JOIN menu_items AS mi\n ON menu_items.id = o.menu_item_id\n INNER JOIN categories AS c\n ON c.id = mi.category_id\n INNER JOIN menu AS m\n ON m.id = c.menu_id\n WHERE o.placed_on >= NOW() - interval '7 days';" }, "description": "Loads newly placed orders weekly.", "latestRun": { "id": "ad6cf477-b233-4305-9bc3-4cc48fe08047", "createdAt": "2022-08-04T05:02:16.660488Z", "updatedAt": "2022-08-04T05:08:09.190723Z", "nominalStartTime": "2022-08-04T05:04:00Z", "nominalEndTime": "2022-08-04T05:08:00Z", "state": "COMPLETED", "startedAt": "2022-08-04T05:04:12.190723Z", "endedAt": "2022-08-04T05:08:09.190723Z", "durationMs": 237000, "args": {}, "jobVersion": { "namespace": "food_delivery", "name": "etl_orders.etl_orders_7_days", "version": "e1cfd2ef-8197-34e4-b361-8ffc012cff81" }, "inputVersions": [{ "namespace": "food_delivery", "name": "public.orders", "version": "42d5a311-e3df-3ab6-bc8b-baed3a5910b0" }, { "namespace": "food_delivery", "name": "public.menu_items", "version": "ee00a374-4a87-3606-8447-b4000aa5090b" }, { "namespace": "food_delivery", "name": "public.menus", "version": "5b4890a4-863c-3a87-8f88-99d64f0d38dc" }, { "namespace": "food_delivery", "name": "public.categories", "version": "d4ea4286-d8f3-3640-bad0-29afa61eb52d" }], "outputVersions": [{ "namespace": "food_delivery", "name": "public.orders_7_days", "version": "3c7f6da4-e79a-3eeb-83c6-dcffe8b9ced9" }, { "namespace": "food_delivery", "name": "public.orders_7_days", "version": "3dc3a809-8bdc-3c50-809d-fa24e5d48455" }], "context": { "job.facets.sourceCodeLocation.url": "https://github.com/DatakinHQ/demo/blob/bd6303a76f4fa432d0853a07bb2e4b6d95bd0c49/airflow/dags/etl_orders_7_days.py", "sql": "INSERT INTO orders_7_days (order_id, placed_on, discount_id, menu_id, restaurant_id, menu_item_id, category_id)\n SELECT o.id AS order_id, o.placed_on, o.discount_id, m.id AS menu_id, m.restaurant_id, mi.id AS menu_item_id, c.id AS category_id\n FROM orders AS o\n INNER JOIN menu_items AS mi\n ON menu_items.id = o.menu_item_id\n INNER JOIN categories AS c\n ON c.id = mi.category_id\n INNER JOIN menu AS m\n ON m.id = c.menu_id\n WHERE o.placed_on >= NOW() - interval '7 days';" }, "facets": { "nominalTime": { "_producer": "com.datakin.cli.SeedCommand", "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/NominalTimeRunFacet.json#/$defs/NominalTimeRunFacet", "nominalEndTime": "2022-08-04T05:08:00Z", "nominalStartTime": "2022-08-04T05:04:00Z" } } } }, "inEdges": [{ "origin": "dataset:food_delivery:public.categories", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }, { "origin": "dataset:food_delivery:public.menu_items", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }, { "origin": "dataset:food_delivery:public.menus", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }, { "origin": "dataset:food_delivery:public.orders", "destination": "job:food_delivery:etl_orders.etl_orders_7_days" }], "outEdges": [{ "origin": "job:food_delivery:etl_orders.etl_orders_7_days", "destination": "dataset:food_delivery:public.orders_7_days" }] }] }Proposed
add column lineage to existing endpoint
by embedding the columnLineage facet in the data section of DATASET nodes
add a column-level-lineage endpoint:
that would be layered on the existing lineage endpoint but filtered down to the datasets that contribute to that column.
It also only returns dataset nodes
Point in time lineage
return historical upstream lineage from a given version.
This returns only upstream lineage in this current proposal.
The upstream lineage is well defined to a specific version while downstream lineage is not
The data payload would also add a version.
The text was updated successfully, but these errors were encountered: