From 4fbd52185ed90210fc0a53d926b437ecdc148910 Mon Sep 17 00:00:00 2001 From: jdpgrailsdev Date: Thu, 6 Oct 2022 16:27:10 -0400 Subject: [PATCH 1/4] Ensure database initialization in test container --- .../db/instance/development/MigrationDevCenter.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java index 9b4d2d89af6b..56b30171fcf1 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java @@ -16,6 +16,9 @@ import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.ext.ScriptUtils; +import org.testcontainers.jdbc.JdbcDatabaseDelegate; +import org.testcontainers.utility.MountableFile; /** * Helper class for migration development. See README for details. @@ -47,6 +50,9 @@ private static PostgreSQLContainer createContainer() { .withUsername("docker") .withPassword("docker"); container.start(); + + initializeDatabase(container); + return container; } @@ -106,6 +112,12 @@ private void dumpSchema() { } } + private static void initializeDatabase(final PostgreSQLContainer container) { + final var containerDelegate = new JdbcDatabaseDelegate(container, ""); + ScriptUtils.runInitScript(containerDelegate, "configs_database/schema.sql"); + ScriptUtils.runInitScript(containerDelegate, "jobs_database/schema.sql"); + } + public static void main(final String[] args) { final MigrationDevCenter devCenter; From 301ce3b9da3f4a237c93b7f5add2adcff41006d9 Mon Sep 17 00:00:00 2001 From: terencecho Date: Thu, 6 Oct 2022 14:46:23 -0700 Subject: [PATCH 2/4] Remove unused import --- .../io/airbyte/db/instance/development/MigrationDevCenter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java index 56b30171fcf1..b95580c9f98b 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/development/MigrationDevCenter.java @@ -18,7 +18,6 @@ import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.ext.ScriptUtils; import org.testcontainers.jdbc.JdbcDatabaseDelegate; -import org.testcontainers.utility.MountableFile; /** * Helper class for migration development. See README for details. From 46707464001ad31854f09a81bfa7ecf1046b0ea4 Mon Sep 17 00:00:00 2001 From: terencecho Date: Thu, 6 Oct 2022 14:46:38 -0700 Subject: [PATCH 3/4] Update schema dump --- .../configs_database/schema_dump.txt | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index 307a091f7966..d091013361c2 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -86,6 +86,25 @@ create table "public"."airbyte_configs_migrations"( constraint "airbyte_configs_migrations_pk" primary key ("installed_rank") ); +create table "public"."airbyte_metadata"( + "key" varchar(255) not null, + "value" varchar(255) null, + constraint "airbyte_metadata_pkey" + primary key ("key") +); +create table "public"."attempts"( + "id" int8 generated by default as identity not null, + "job_id" int8 null, + "attempt_number" int4 null, + "log_path" varchar(255) null, + "output" jsonb null, + "status" any null, + "created_at" timestamptz(35) null, + "updated_at" timestamptz(35) null, + "ended_at" timestamptz(35) null, + constraint "attempts_pkey" + primary key ("id") +); create table "public"."connection"( "id" uuid not null, "namespace_definition" namespace_definition_type not null, @@ -121,6 +140,18 @@ create table "public"."connection_operation"( "operation_id" ) ); +create table "public"."jobs"( + "id" int8 generated by default as identity not null, + "config_type" any null, + "scope" varchar(255) null, + "config" jsonb null, + "status" any null, + "started_at" timestamptz(35) null, + "created_at" timestamptz(35) null, + "updated_at" timestamptz(35) null, + constraint "jobs_pkey" + primary key ("id") +); create table "public"."operation"( "id" uuid not null, "workspace_id" uuid not null, @@ -268,6 +299,12 @@ create index "actor_oauth_parameter_workspace_definition_idx" on "public"."actor ); create unique index "airbyte_configs_migrations_pk" on "public"."airbyte_configs_migrations"("installed_rank" asc); create index "airbyte_configs_migrations_s_idx" on "public"."airbyte_configs_migrations"("success" asc); +create unique index "airbyte_metadata_pkey" on "public"."airbyte_metadata"("key" asc); +create unique index "attempts_pkey" on "public"."attempts"("id" asc); +create unique index "job_attempt_idx" on "public"."attempts"( + "job_id" asc, + "attempt_number" asc +); create index "connection_destination_id_idx" on "public"."connection"("destination_id" asc); create unique index "connection_pkey" on "public"."connection"("id" asc); create index "connection_source_id_idx" on "public"."connection"("source_id" asc); @@ -277,6 +314,7 @@ create unique index "connection_operation_pkey" on "public"."connection_operatio "connection_id" asc, "operation_id" asc ); +create unique index "jobs_pkey" on "public"."jobs"("id" asc); create unique index "operation_pkey" on "public"."operation"("id" asc); create unique index "state__connection_id__stream_name__namespace__uq" on "public"."state"( "connection_id" asc, From 6c6c9deb14a80e7302b691b0ba9bc51f0a0ece7a Mon Sep 17 00:00:00 2001 From: jdpgrailsdev Date: Fri, 7 Oct 2022 10:42:49 -0400 Subject: [PATCH 4/4] Add schema dump --- .../main/resources/jobs_database/schema_dump.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt index 4129b16d6f27..d555514a2c27 100644 --- a/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt @@ -2,6 +2,16 @@ // It is also not used by any piece of code to generate anything. // It doesn't contain the enums created in the database and the default values might also be buggy. +create table "public"."airbyte_configs"( + "id" int8 generated by default as identity not null, + "config_id" varchar(36) not null, + "config_type" varchar(60) not null, + "config_blob" jsonb not null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "airbyte_configs_pkey" + primary key ("id") +); create table "public"."airbyte_jobs_migrations"( "installed_rank" int4 not null, "version" varchar(50) null, @@ -85,6 +95,12 @@ alter table "public"."sync_stats" add constraint "sync_stats_attempt_id_fkey" foreign key ("attempt_id") references "public"."attempts" ("id"); +create index "airbyte_configs_id_idx" on "public"."airbyte_configs"("config_id" asc); +create unique index "airbyte_configs_pkey" on "public"."airbyte_configs"("id" asc); +create unique index "airbyte_configs_type_id_idx" on "public"."airbyte_configs"( + "config_type" asc, + "config_id" asc +); create unique index "airbyte_jobs_migrations_pk" on "public"."airbyte_jobs_migrations"("installed_rank" asc); create index "airbyte_jobs_migrations_s_idx" on "public"."airbyte_jobs_migrations"("success" asc); create unique index "airbyte_metadata_pkey" on "public"."airbyte_metadata"("key" asc);