Merge branch 'master' into ddavydov/#45-stripe-implement-slicing

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.39.41-alpha
+current_version = 0.40.0-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.39.41-alpha
+VERSION=0.40.0-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -100,4 +100,4 @@ METRIC_CLIENT=
 # Useful only when metric client is set to be otel. Must start with http:// or https://.
 OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"
 
-USE_STREAM_CAPABLE_STATE=false
+USE_STREAM_CAPABLE_STATE=true

.github/workflows/gradle.yml

@@ -234,6 +234,15 @@ jobs:
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
 
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v1
+        if: always()
+        with:
+          name: Connectors Base Test Report
+          # Specify top-level and second-level modules. Note there cannot be a space between the comma.
+          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml'
+          reporter: java-junit
+
   # In case of self-hosted EC2 errors, remove this block.
   stop-connectors-base-build-runner:
     name: "Connectors Base: Stop Build EC2 Runner"
@@ -471,7 +480,8 @@ jobs:
           SUB_BUILD=PLATFORM ./gradlew build javadoc --scan
 
       - name: Integration test
-        run: SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
+        run: |
+          SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
 
       - name: Slow integration test
         if: contains(github.ref, 'bump-version') || contains(github.ref, 'master')
@@ -485,16 +495,25 @@ jobs:
         if: success() && github.ref == 'refs/heads/master'
         run: ./tools/site/link_checker.sh check_docs
 
-      # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
-      #      - name: Get Docker Space
-      #        run: docker run --rm busybox df -h
+#       This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
+#      - name: Get Docker Space
+#        run: docker run --rm busybox df -h
 
       - name: Run End-to-End Acceptance Tests
         run: ./tools/bin/acceptance_test.sh
 
       - name: Automatic Migration Acceptance Test
         run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
 
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v1
+        if: always()    # run this step even if previous step failed
+        with:
+          name: Platform Test Report with Docker E2E Test
+          # Specify top-level and second-level modules. Note there cannot be a space between the comma.
+          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte/airbyte/*/*/build/test-results/*/*.xml'
+          reporter: java-junit
+
   # In case of self-hosted EC2 errors, remove this block.
   stop-platform-build-runner:
     name: "Platform: Stop Build EC2 Runner"
@@ -626,6 +645,14 @@ jobs:
         run: |
           CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
 
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v1
+        if: always()    # run this step even if previous step failed
+        with:
+          name: Platform Kubernetes E2E Test Report
+          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
+          reporter: java-junit
+
       - uses: actions/upload-artifact@v2
         if: failure()
         with:

.github/workflows/shared-pulls.yml

@@ -4,13 +4,35 @@ on:
     types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened]
 
 jobs:
+  find_valid_pat:
+    name: "Find a PAT with room for actions"
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      pat: ${{ steps.variables.outputs.pat }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        id: variables
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.OCTAVIA_PAT }} \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
+            ${{ secrets.SUPERTOPHER_PAT }} \
+            ${{ secrets.DAVINCHIA_PAT }}
+
   shared-pulls:
+    name: "Label github issues for tracking"
+    needs:
+      - find_valid_pat
     runs-on: ubuntu-latest
     steps:
       - uses: nick-fields/private-action-loader@v3
         with:
-          pal-repo-token: "${{ secrets.OCTAVIA_PAT }}"
+          pal-repo-token: ${{ needs.find_valid_pat.outputs.pat }}
           pal-repo-name: airbytehq/workflow-actions@production
           # the following input gets passed to the private action
-          token: "${{ secrets.OCTAVIA_PAT }}"
+          token: ${{ needs.find_valid_pat.outputs.pat }}
           command: "pull"

README.md

@@ -44,7 +44,7 @@ Here is a [step-by-step guide](https://github.com/airbytehq/airbyte/tree/e378d40
 
 We love contributions to Airbyte, big or small.
 
-See our [Contributing guide](docs/09-contributing-to-airbyte/README.md) on how to get started. Not sure where to start? We’ve listed some [good first issues](https://github.com/airbytehq/airbyte/labels/good%20first%20issue) to start with. If you have any questions, please open a draft PR or visit our [slack channel](https://github.com/airbytehq/airbyte/tree/a9b1c6c0420550ad5069aca66c295223e0d05e27/slack.airbyte.io) where the core team can help answer your questions.
+See our [Contributing guide](docs/contributing-to-airbyte/README.md) on how to get started. Not sure where to start? We’ve listed some [good first issues](https://github.com/airbytehq/airbyte/labels/good%20first%20issue) to start with. If you have any questions, please open a draft PR or visit our [slack channel](https://github.com/airbytehq/airbyte/tree/a9b1c6c0420550ad5069aca66c295223e0d05e27/slack.airbyte.io) where the core team can help answer your questions.
 
 **Note that you are able to create connectors using the language you want, as Airbyte connections run as Docker containers.**
 

airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java

@@ -35,6 +35,8 @@ class SegmentTrackingClientTest {
   private static final TrackingIdentity IDENTITY = new TrackingIdentity(AIRBYTE_VERSION, UUID.randomUUID(), EMAIL, false, false, true);
   private static final UUID WORKSPACE_ID = UUID.randomUUID();
   private static final Function<UUID, TrackingIdentity> MOCK_TRACKING_IDENTITY = (workspaceId) -> IDENTITY;
+  private static final String AIRBYTE_VERSION_KEY = "airbyte_version";
+  private static final String JUMP = "jump";
 
   private Analytics analytics;
   private SegmentTrackingClient segmentTrackingClient;
@@ -61,7 +63,7 @@ void testIdentify() {
     final IdentifyMessage actual = mockBuilder.getValue().build();
     final Map<String, Object> expectedTraits = ImmutableMap.<String, Object>builder()
         .put("anonymized", IDENTITY.isAnonymousDataCollection())
-        .put("airbyte_version", AIRBYTE_VERSION.serialize())
+        .put(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize())
         .put("deployment_env", DEPLOYMENT.getDeploymentEnv())
         .put("deployment_mode", DEPLOYMENT.getDeploymentMode())
         .put("deployment_id", DEPLOYMENT.getDeploymentId())
@@ -87,7 +89,7 @@ void testIdentifyWithRole() {
     final IdentifyMessage actual = mockBuilder.getValue().build();
     final Map<String, Object> expectedTraits = ImmutableMap.<String, Object>builder()
         .put("airbyte_role", "role")
-        .put("airbyte_version", AIRBYTE_VERSION.serialize())
+        .put(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize())
         .put("anonymized", IDENTITY.isAnonymousDataCollection())
         .put("deployment_env", DEPLOYMENT.getDeploymentEnv())
         .put("deployment_mode", DEPLOYMENT.getDeploymentMode())
@@ -104,13 +106,13 @@ void testIdentifyWithRole() {
   void testTrack() {
     final ArgumentCaptor<TrackMessage.Builder> mockBuilder = ArgumentCaptor.forClass(TrackMessage.Builder.class);
     final ImmutableMap<String, Object> metadata =
-        ImmutableMap.of("airbyte_version", AIRBYTE_VERSION.serialize(), "user_id", IDENTITY.getCustomerId());
+        ImmutableMap.of(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize(), "user_id", IDENTITY.getCustomerId());
 
-    segmentTrackingClient.track(WORKSPACE_ID, "jump");
+    segmentTrackingClient.track(WORKSPACE_ID, JUMP);
 
     verify(analytics).enqueue(mockBuilder.capture());
     final TrackMessage actual = mockBuilder.getValue().build();
-    assertEquals("jump", actual.event());
+    assertEquals(JUMP, actual.event());
     assertEquals(IDENTITY.getCustomerId().toString(), actual.userId());
     assertEquals(metadata, filterTrackedAtProperty(Objects.requireNonNull(actual.properties())));
   }
@@ -119,16 +121,16 @@ void testTrack() {
   void testTrackWithMetadata() {
     final ArgumentCaptor<TrackMessage.Builder> mockBuilder = ArgumentCaptor.forClass(TrackMessage.Builder.class);
     final ImmutableMap<String, Object> metadata = ImmutableMap.of(
-        "airbyte_version", AIRBYTE_VERSION.serialize(),
+        AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize(),
         "email", EMAIL,
         "height", "80 meters",
         "user_id", IDENTITY.getCustomerId());
 
-    segmentTrackingClient.track(WORKSPACE_ID, "jump", metadata);
+    segmentTrackingClient.track(WORKSPACE_ID, JUMP, metadata);
 
     verify(analytics).enqueue(mockBuilder.capture());
     final TrackMessage actual = mockBuilder.getValue().build();
-    assertEquals("jump", actual.event());
+    assertEquals(JUMP, actual.event());
     assertEquals(IDENTITY.getCustomerId().toString(), actual.userId());
     assertEquals(metadata, filterTrackedAtProperty(Objects.requireNonNull(actual.properties())));
   }

airbyte-bootloader/Dockerfile

@@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye
 ARG JDK_IMAGE=openjdk:${JDK_VERSION}
 FROM ${JDK_IMAGE}
 
-ARG VERSION=0.39.41-alpha
+ARG VERSION=0.40.0-alpha
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

@@ -59,13 +59,18 @@ class BootloaderAppTest {
   private PostgreSQLContainer container;
   private DataSource configsDataSource;
   private DataSource jobsDataSource;
+  private static final String DOCKER = "docker";
+  private static final String VERSION_0330_ALPHA = "0.33.0-alpha";
+  private static final String VERSION_0320_ALPHA = "0.32.0-alpha";
+  private static final String VERSION_0321_ALPHA = "0.32.1-alpha";
+  private static final String VERSION_0170_ALPHA = "0.17.0-alpha";
 
   @BeforeEach
   void setup() {
     container = new PostgreSQLContainer<>("postgres:13-alpine")
         .withDatabaseName("public")
-        .withUsername("docker")
-        .withPassword("docker");
+        .withUsername(DOCKER)
+        .withPassword(DOCKER);
     container.start();
 
     configsDataSource =
@@ -86,16 +91,14 @@ void cleanup() throws Exception {
 
   @Test
   void testBootloaderAppBlankDb() throws Exception {
-    val version = "0.33.0-alpha";
-
     val mockedConfigs = mock(Configs.class);
     when(mockedConfigs.getConfigDatabaseUrl()).thenReturn(container.getJdbcUrl());
     when(mockedConfigs.getConfigDatabaseUser()).thenReturn(container.getUsername());
     when(mockedConfigs.getConfigDatabasePassword()).thenReturn(container.getPassword());
     when(mockedConfigs.getDatabaseUrl()).thenReturn(container.getJdbcUrl());
     when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername());
     when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword());
-    when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version));
+    when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA));
     when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true);
     when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L);
     when(mockedConfigs.getJobsDatabaseInitializationTimeoutMs()).thenReturn(60000L);
@@ -107,8 +110,8 @@ void testBootloaderAppBlankDb() throws Exception {
     // Although we are able to inject mocked configs into the Bootloader, a particular migration in the
     // configs database
     // requires the env var to be set. Flyway prevents injection, so we dynamically set this instead.
-    environmentVariables.set("DATABASE_USER", "docker");
-    environmentVariables.set("DATABASE_PASSWORD", "docker");
+    environmentVariables.set("DATABASE_USER", DOCKER);
+    environmentVariables.set("DATABASE_PASSWORD", DOCKER);
     environmentVariables.set("DATABASE_URL", container.getJdbcUrl());
 
     try (val configsDslContext = DSLContextFactory.create(configsDataSource, SQLDialect.POSTGRES);
@@ -133,24 +136,22 @@ void testBootloaderAppBlankDb() throws Exception {
       assertEquals("0.39.17.001", configsMigrator.getLatestMigration().getVersion().getVersion());
 
       val jobsPersistence = new DefaultJobPersistence(jobDatabase);
-      assertEquals(version, jobsPersistence.getVersion().get());
+      assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get());
 
       assertNotEquals(Optional.empty(), jobsPersistence.getDeployment().get());
     }
   }
 
   @Test
   void testBootloaderAppRunSecretMigration() throws Exception {
-    val version = "0.33.0-alpha";
-
     val mockedConfigs = mock(Configs.class);
     when(mockedConfigs.getConfigDatabaseUrl()).thenReturn(container.getJdbcUrl());
     when(mockedConfigs.getConfigDatabaseUser()).thenReturn(container.getUsername());
     when(mockedConfigs.getConfigDatabasePassword()).thenReturn(container.getPassword());
     when(mockedConfigs.getDatabaseUrl()).thenReturn(container.getJdbcUrl());
     when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername());
     when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword());
-    when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version));
+    when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA));
     when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true);
     when(mockedConfigs.getSecretPersistenceType()).thenReturn(TESTING_CONFIG_DB_TABLE);
     when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L);
@@ -181,8 +182,8 @@ void testBootloaderAppRunSecretMigration() throws Exception {
       // Although we are able to inject mocked configs into the Bootloader, a particular migration in the
       // configs database requires the env var to be set. Flyway prevents injection, so we dynamically set
       // this instead.
-      environmentVariables.set("DATABASE_USER", "docker");
-      environmentVariables.set("DATABASE_PASSWORD", "docker");
+      environmentVariables.set("DATABASE_USER", DOCKER);
+      environmentVariables.set("DATABASE_PASSWORD", DOCKER);
       environmentVariables.set("DATABASE_URL", container.getJdbcUrl());
 
       // Bootstrap the database for the test
@@ -265,37 +266,35 @@ void testBootloaderAppRunSecretMigration() throws Exception {
   void testIsLegalUpgradePredicate() {
     // starting from no previous version is always legal.
     assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.17.1-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.32.0-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.32.1-alpha")));
+    assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion(VERSION_0320_ALPHA)));
+    assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion(VERSION_0321_ALPHA)));
     assertTrue(BootloaderApp.isLegalUpgrade(null, new AirbyteVersion("0.33.1-alpha")));
     // starting from a version that is pre-breaking migration cannot go past the breaking migration.
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.17.1-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.18.0-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.32.0-alpha")));
-    assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.32.1-alpha")));
-    assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.17.0-alpha"), new AirbyteVersion("0.33.0-alpha")));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion("0.17.1-alpha")));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion("0.18.0-alpha")));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion(VERSION_0320_ALPHA)));
+    assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion(VERSION_0321_ALPHA)));
+    assertFalse(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0170_ALPHA), new AirbyteVersion(VERSION_0330_ALPHA)));
     // any migration starting at the breaking migration or after it can upgrade to anything.
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.0-alpha"), new AirbyteVersion("0.32.1-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.0-alpha"), new AirbyteVersion("0.33.0-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.1-alpha"), new AirbyteVersion("0.32.1-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.32.1-alpha"), new AirbyteVersion("0.33.0-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.33.0-alpha"), new AirbyteVersion("0.33.1-alpha")));
-    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion("0.33.0-alpha"), new AirbyteVersion("0.34.0-alpha")));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0320_ALPHA), new AirbyteVersion(VERSION_0321_ALPHA)));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0320_ALPHA), new AirbyteVersion(VERSION_0330_ALPHA)));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0321_ALPHA), new AirbyteVersion(VERSION_0321_ALPHA)));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0321_ALPHA), new AirbyteVersion(VERSION_0330_ALPHA)));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0330_ALPHA), new AirbyteVersion("0.33.1-alpha")));
+    assertTrue(BootloaderApp.isLegalUpgrade(new AirbyteVersion(VERSION_0330_ALPHA), new AirbyteVersion("0.34.0-alpha")));
   }
 
   @Test
   void testPostLoadExecutionExecutes() throws Exception {
     final var testTriggered = new AtomicBoolean();
-    val version = "0.33.0-alpha";
-
     val mockedConfigs = mock(Configs.class);
     when(mockedConfigs.getConfigDatabaseUrl()).thenReturn(container.getJdbcUrl());
     when(mockedConfigs.getConfigDatabaseUser()).thenReturn(container.getUsername());
     when(mockedConfigs.getConfigDatabasePassword()).thenReturn(container.getPassword());
     when(mockedConfigs.getDatabaseUrl()).thenReturn(container.getJdbcUrl());
     when(mockedConfigs.getDatabaseUser()).thenReturn(container.getUsername());
     when(mockedConfigs.getDatabasePassword()).thenReturn(container.getPassword());
-    when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(version));
+    when(mockedConfigs.getAirbyteVersion()).thenReturn(new AirbyteVersion(VERSION_0330_ALPHA));
     when(mockedConfigs.runDatabaseMigrationOnStartup()).thenReturn(true);
     when(mockedConfigs.getConfigsDatabaseInitializationTimeoutMs()).thenReturn(60000L);
     when(mockedConfigs.getJobsDatabaseInitializationTimeoutMs()).thenReturn(60000L);