From f5c72a8dbde5e4faae3070963afafbb927ae6f30 Mon Sep 17 00:00:00 2001 From: wslulciuc Date: Wed, 30 Aug 2023 16:41:22 -0700 Subject: [PATCH 1/2] Postgres 14 Signed-off-by: wslulciuc --- .circleci/db-migration.sh | 2 +- README.md | 2 +- api/src/test/java/marquez/PostgresContainer.java | 2 +- api/src/test/java/marquez/db/DbTest.java | 2 +- docker-compose.db.yml | 2 +- docker-compose.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/db-migration.sh b/.circleci/db-migration.sh index 3c535cb73e..ab4c109c57 100755 --- a/.circleci/db-migration.sh +++ b/.circleci/db-migration.sh @@ -11,7 +11,7 @@ # Usage: $ ./db-migration.sh # Version of PostgreSQL -readonly POSTGRES_VERSION="12.1" +readonly POSTGRES_VERSION="14" # Version of Marquez readonly MARQUEZ_VERSION=0.40.0 # Build version of Marquez diff --git a/README.md b/README.md index 2961483fbd..4e68a53165 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Marquez uses a _multi_-project structure and contains the following modules: ## Requirements * [Java 17](https://adoptium.net) -* [PostgreSQL 12.1](https://www.postgresql.org/download) +* [PostgreSQL 14](https://www.postgresql.org/download) > **Note:** To connect to your running PostgreSQL instance, you will need the standard [`psql`](https://www.postgresql.org/docs/9.6/app-psql.html) tool. diff --git a/api/src/test/java/marquez/PostgresContainer.java b/api/src/test/java/marquez/PostgresContainer.java index 7e1a250d2d..0d2b4793c3 100644 --- a/api/src/test/java/marquez/PostgresContainer.java +++ b/api/src/test/java/marquez/PostgresContainer.java @@ -11,7 +11,7 @@ import org.testcontainers.utility.DockerImageName; public final class PostgresContainer extends PostgreSQLContainer { - private static final DockerImageName POSTGRES = DockerImageName.parse("postgres:12.12"); + private static final DockerImageName POSTGRES = DockerImageName.parse("postgres:14"); private static final int JDBC = 5; private static final Map containers = new HashMap<>(); diff --git a/api/src/test/java/marquez/db/DbTest.java b/api/src/test/java/marquez/db/DbTest.java index 15ec7e2c31..c1571f461b 100644 --- a/api/src/test/java/marquez/db/DbTest.java +++ b/api/src/test/java/marquez/db/DbTest.java @@ -33,7 +33,7 @@ @Tag("DataAccessTests") @Testcontainers class DbTest { - private static final DockerImageName POSTGRES_12_1 = DockerImageName.parse("postgres:12.1"); + private static final DockerImageName POSTGRES_12_1 = DockerImageName.parse("postgres:14"); @Container private static final PostgreSQLContainer DB_CONTAINER = diff --git a/docker-compose.db.yml b/docker-compose.db.yml index 2fcafc40d7..77cfa1e9fb 100644 --- a/docker-compose.db.yml +++ b/docker-compose.db.yml @@ -1,7 +1,7 @@ version: "3.7" services: db: - image: postgres:12.1 + image: postgres:14 container_name: marquez-db ports: - "5432:5432" diff --git a/docker-compose.yml b/docker-compose.yml index 1d54438684..bf275001be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: entrypoint: ["/opt/marquez/wait-for-it.sh", "db:5432", "--", "./entrypoint.sh"] db: - image: postgres:12.1 + image: postgres:14 container_name: marquez-db ports: - "5432:5432" From 29a5057745189edd77dbde0fac92b54ba13e6d45 Mon Sep 17 00:00:00 2001 From: Pawel Leszczynski Date: Wed, 6 Sep 2023 09:41:04 +0200 Subject: [PATCH 2/2] fix migration backfill for psql14 Signed-off-by: Pawel Leszczynski --- .../marquez/db/migrations/V57_1__BackfillFacets.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/api/src/main/java/marquez/db/migrations/V57_1__BackfillFacets.java b/api/src/main/java/marquez/db/migrations/V57_1__BackfillFacets.java index bc76e591fd..f51ba29d02 100644 --- a/api/src/main/java/marquez/db/migrations/V57_1__BackfillFacets.java +++ b/api/src/main/java/marquez/db/migrations/V57_1__BackfillFacets.java @@ -128,6 +128,18 @@ public void migrate(Context context) throws Exception { } int estimatedEventsCount = estimateCountLineageEvents(); + + if (estimatedEventsCount < 0) { + // https://www.postgresql.org/docs/current/catalog-pg-class.html + // -1 indicating that the row count is unknown. + // This happens when lineage_events table is empty. + log.info("Vacuuming lineage_events table"); + jdbi.withHandle(h -> h.execute("VACUUM lineage_events;")); + log.info("Vacuuming lineage_events table finished"); + + estimatedEventsCount = estimateCountLineageEvents(); + } + log.info("Estimating {} events in lineage_events table", estimatedEventsCount); if (estimatedEventsCount == 0 && countLineageEvents() == 0) { // lineage_events table is empty -> no need to run migration