From 24e07eb412f68e34ba7cf6df2947fdeb0ed006c2 Mon Sep 17 00:00:00 2001 From: Prachi Mishra Date: Wed, 7 Dec 2022 21:45:12 -0800 Subject: [PATCH 1/5] adding indexed created_at column to lineage events table Signed-off-by: Prachi Mishra --- .../db/migration/V52__lineage_events_created_at_indexed.sql | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql diff --git a/api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql b/api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql new file mode 100644 index 0000000000..347697b890 --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql @@ -0,0 +1,4 @@ +ALTER TABLE lineage_events ADD created_at TIMESTAMP; + +create index lineage_events_created_at_index + on lineage_events (created_at desc); \ No newline at end of file From 67e76d219f1f9b08bc75d2945e5291b5a155f566 Mon Sep 17 00:00:00 2001 From: Prachi Mishra Date: Thu, 8 Dec 2022 16:50:59 -0800 Subject: [PATCH 2/5] populate default values in lineage_events.created_at column Signed-off-by: Prachi Mishra --- .../migration/V52__lineage_events_created_at_indexed.sql | 4 ---- .../migration/V54__lineage_events_created_at_indexed.sql | 9 +++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql create mode 100644 api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql diff --git a/api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql b/api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql deleted file mode 100644 index 347697b890..0000000000 --- a/api/src/main/resources/marquez/db/migration/V52__lineage_events_created_at_indexed.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE lineage_events ADD created_at TIMESTAMP; - -create index lineage_events_created_at_index - on lineage_events (created_at desc); \ No newline at end of file diff --git a/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql new file mode 100644 index 0000000000..213bf1dc8a --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql @@ -0,0 +1,9 @@ +ALTER TABLE lineage_events ADD created_at TIMESTAMP; + +create index lineage_events_created_at_index on lineage_events (created_at desc); + +UPDATE lineage_events +SET created_at = (event_time AT TIME ZONE 'UTC')::timestamp; + +ALTER TABLE lineage_events ALTER COLUMN created_at SET DEFAULT (now() AT TIME ZONE 'UTC'); + From 60ede8c84bffa684f3c29fad823f64c7f0078215 Mon Sep 17 00:00:00 2001 From: Prachi Mishra Date: Fri, 9 Dec 2022 15:37:28 -0800 Subject: [PATCH 3/5] removing backfill step Signed-off-by: Prachi Mishra --- .../migration/V54__lineage_events_created_at_indexed.sql | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql index 213bf1dc8a..04cfd3ef94 100644 --- a/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql +++ b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql @@ -1,9 +1,7 @@ +-- new column will be created with 'null' filled for existing rows ALTER TABLE lineage_events ADD created_at TIMESTAMP; create index lineage_events_created_at_index on lineage_events (created_at desc); -UPDATE lineage_events -SET created_at = (event_time AT TIME ZONE 'UTC')::timestamp; - -ALTER TABLE lineage_events ALTER COLUMN created_at SET DEFAULT (now() AT TIME ZONE 'UTC'); - +-- The new default set to UTC now() will only apply in subsequent INSERT or UPDATE commands; it does not cause rows already in the table to change. +ALTER TABLE lineage_events ALTER COLUMN created_at SET DEFAULT (now() AT TIME ZONE 'UTC'); \ No newline at end of file From 692b8aa8ea45eccff2b202b0e3f5828cae5e44a7 Mon Sep 17 00:00:00 2001 From: Prachi Mishra Date: Fri, 9 Dec 2022 16:00:11 -0800 Subject: [PATCH 4/5] altering type to TIMESTAMP WITH TIME ZONE Signed-off-by: Prachi Mishra --- .../db/migration/V54__lineage_events_created_at_indexed.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql index 04cfd3ef94..d9c7208014 100644 --- a/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql +++ b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql @@ -1,7 +1,7 @@ -- new column will be created with 'null' filled for existing rows -ALTER TABLE lineage_events ADD created_at TIMESTAMP; +ALTER TABLE lineage_events ADD created_at TIMESTAMP WITH TIME ZONE; create index lineage_events_created_at_index on lineage_events (created_at desc); -- The new default set to UTC now() will only apply in subsequent INSERT or UPDATE commands; it does not cause rows already in the table to change. -ALTER TABLE lineage_events ALTER COLUMN created_at SET DEFAULT (now() AT TIME ZONE 'UTC'); \ No newline at end of file +ALTER TABLE lineage_events ALTER COLUMN created_at SET DEFAULT (now() AT TIME ZONE 'UTC')::timestamptz; \ No newline at end of file From dc70917d6f09375267e9fb548ee5aac63f17c3e4 Mon Sep 17 00:00:00 2001 From: Prachi Mishra Date: Mon, 12 Dec 2022 13:01:50 -0800 Subject: [PATCH 5/5] adding NULLS LAST in desc index on lineage_events.created_at Signed-off-by: Prachi Mishra --- .../db/migration/V54__lineage_events_created_at_indexed.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql index d9c7208014..cd085feb5d 100644 --- a/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql +++ b/api/src/main/resources/marquez/db/migration/V54__lineage_events_created_at_indexed.sql @@ -1,7 +1,7 @@ -- new column will be created with 'null' filled for existing rows ALTER TABLE lineage_events ADD created_at TIMESTAMP WITH TIME ZONE; -create index lineage_events_created_at_index on lineage_events (created_at desc); +create index lineage_events_created_at_index on lineage_events (created_at desc NULLS LAST); -- The new default set to UTC now() will only apply in subsequent INSERT or UPDATE commands; it does not cause rows already in the table to change. ALTER TABLE lineage_events ALTER COLUMN created_at SET DEFAULT (now() AT TIME ZONE 'UTC')::timestamptz; \ No newline at end of file