airbyte-ci: re-enable caching on source-postgres CI (#32427)

airbytehq · Nov 13, 2023 · 7973311 · 7973311
1 parent 868083c
commit 7973311
Show file tree

Hide file tree

Showing 5 changed files with 2 additions and 115 deletions.
diff --git a/.github/workflows/tmp-source-postgres-test.yml b/.github/workflows/tmp-source-postgres-test.yml
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -408,6 +408,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 2.5.9   | [#32427](https://github.com/airbytehq/airbyte/pull/32427)  | Re-enable caching for source-postgres                                                                 |
 | 2.5.8   | [#32402](https://github.com/airbytehq/airbyte/pull/32402)  | Set Dagger Cloud token for airbyters only                                                                 |
 | 2.5.7   | [#31628](https://github.com/airbytehq/airbyte/pull/31628)  | Add ClickPipelineContext class                                                                            |
 | 2.5.6   | [#32139](https://github.com/airbytehq/airbyte/pull/32139)  | Test coverage report on Python connector UnitTest.                                                        |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py
@@ -7,7 +7,6 @@
 
 import pipelines.dagger.actions.system.docker
 from dagger import CacheSharingMode, CacheVolume
-from pipelines import hacks
 from pipelines.consts import AMAZONCORRETTO_IMAGE
 from pipelines.dagger.actions import secrets
 from pipelines.helpers.utils import sh_dash_c
@@ -125,8 +124,6 @@ async def _run(self) -> StepResult:
             .with_env_variable("TESTCONTAINERS_RYUK_DISABLED", "true")
             # Set the current working directory.
             .with_workdir("/airbyte")
-            # TODO: remove this once we finish the project to boost source-postgres CI performance.
-            .with_env_variable("CACHEBUSTER", hacks.get_cachebuster(self.context, self.logger))
         )
 
         # Augment the base container with S3 build cache secrets when available.

diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py
@@ -6,7 +6,6 @@
 
 from __future__ import annotations
 
-from logging import Logger
 from typing import TYPE_CHECKING, Callable, List
 
 from pipelines import consts
@@ -71,33 +70,3 @@ def never_fail_exec_inner(container: Container):
         return container.with_exec(["sh", "-c", f"{' '.join(command)}; echo $? > /exit_code"], skip_entrypoint=True)
 
     return never_fail_exec_inner
-
-
-# We want to invalidate the persisted dagger cache and gradle cache for source-postgres.
-# We do it in the context of a project to boost the CI speed for this connector.
-# Invalidating the cache on every run will help us gather unbiased metrics on the CI speed.
-# This should be removed once the project is over.
-CONNECTORS_WITHOUT_CACHING = [
-    "source-postgres",
-]
-
-
-def get_cachebuster(context: ConnectorContext, logger: Logger) -> str:
-    """
-    This function will return a semi-static cachebuster value for connectors in CONNECTORS_WITHOUT_CACHING and a static value for all other connectors.
-    By semi-static I mean that the value (the pipeline start time) will change on each pipeline execution but will be the same for all the steps of the pipeline.
-    It ensures we do not use the remotely persisted dagger cache but we still benefit from the buildkit layer caching inside the pipeline execution.
-    This hack is useful to collect unbiased metrics on the CI speed for connectors in CONNECTORS_WITHOUT_CACHING.
-
-    When the cachebuster value is static it won't invalidate the dagger cache because it's the same value as the previous run: no layer will be rebuilt.
-    When the cachebuster value is changed it will invalidate the dagger cache because it's a different value than the previous run: all downstream layers will be rebuilt.
-
-    Returns:
-        str: The cachebuster value.
-    """
-    if context.connector.technical_name in CONNECTORS_WITHOUT_CACHING:
-        logger.warning(
-            f"Invalidating the persisted dagger cache for {context.connector.technical_name}. Only used in the context of the CI performance improvements project for {context.connector.technical_name}."
-        )
-        return str(context.pipeline_start_timestamp)
-    return "0"
diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "2.5.8"
+version = "2.5.9"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]