From c35ba213769297bc9d73d43cafa11329e9a7ceb0 Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Tue, 21 Mar 2023 21:21:32 +0100
Subject: [PATCH 01/18] connectors-ci: only send commit status on PR

---
 .../ci_connector_ops/pipelines/connectors_ci.py             | 6 +++---
 .../ci_connector_ops/ci_connector_ops/pipelines/contexts.py | 6 +++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index eb6231c6e497..4354f9e121db 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -120,7 +120,7 @@ def connectors_ci(
         ctx.obj["gha_workflow_run_url"],
         GITHUB_GLOBAL_DESCRIPTION,
         GITHUB_GLOBAL_CONTEXT,
-        should_send=not ctx.obj["is_local"],
+        should_send=ctx.obj["ci_context"] == "pull_request",
         logger=logger,
     )
 
@@ -193,7 +193,7 @@ def test_connectors(ctx: click.Context, names: Tuple[str], languages: Tuple[Conn
             ctx.obj["gha_workflow_run_url"],
             GITHUB_GLOBAL_DESCRIPTION,
             GITHUB_GLOBAL_CONTEXT,
-            should_send=not ctx.obj["is_local"],
+            should_send=ctx["ci_context"] == "pull_request",
             logger=logger,
         )
     except dagger.DaggerError as e:
@@ -204,7 +204,7 @@ def test_connectors(ctx: click.Context, names: Tuple[str], languages: Tuple[Conn
             ctx.obj["gha_workflow_run_url"],
             GITHUB_GLOBAL_DESCRIPTION,
             GITHUB_GLOBAL_CONTEXT,
-            should_send=not ctx.obj["is_local"],
+            should_send=ctx["ci_context"] == "pull_request",
             logger=logger,
         )
 
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py
index 7c3e8182da7b..4180cc018ae0 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py
@@ -89,6 +89,10 @@ def dagger_client(self, dagger_client: Client):
     def is_ci(self):
         return self.is_local is False
 
+    @property
+    def is_pr(self):
+        return self.ci_context == "pull_request"
+
     @property
     def repo(self):
         return self.dagger_client.git(AIRBYTE_REPO_URL, keep_git_dir=True)
@@ -122,7 +126,7 @@ def github_commit_status(self) -> dict:
             "target_url": self.gha_workflow_run_url,
             "description": self.state.value["description"],
             "context": f"[POC please ignore] Connector tests: {self.connector.technical_name}",
-            "should_send": self.is_ci,
+            "should_send": self.is_pr,
             "logger": self.logger,
         }
 

From 48d599793a7bdd5ab021eeae00f17666c32771bd Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Tue, 21 Mar 2023 21:25:59 +0100
Subject: [PATCH 02/18] increase timeout

---
 .github/workflows/connector_nightly_builds_dagger.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index bf8af4ed8fd9..b4f89991a764 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -9,7 +9,7 @@ on:
 jobs:
   connectors_ci:
     name: Connectors CI
-    timeout-minutes: 240 # 4 hours
+    timeout-minutes: 300 # 5 hours
     runs-on: large-runner
     steps:
       - name: Get start timestamp

From 6e0c4f3bce77f55d6fb426aaf7fc818fec17e889 Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Tue, 21 Mar 2023 22:53:19 +0100
Subject: [PATCH 03/18] try semaphore

---
 .../connector_nightly_builds_dagger.yml       |  8 ++---
 .../pipelines/connectors_ci.py                |  3 +-
 .../ci_connector_ops/pipelines/tests.py       | 33 ++++++++++---------
 3 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index b4f89991a764..430ad5e62fa2 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -7,9 +7,9 @@ on:
   workflow_dispatch:
 
 jobs:
-  connectors_ci:
-    name: Connectors CI
-    timeout-minutes: 300 # 5 hours
+  nightly_builds_alpha:
+    name: Nightly builds GA + BETA
+    timeout-minutes: 360 # 6 hours
     runs-on: large-runner
     steps:
       - name: Get start timestamp
@@ -41,7 +41,7 @@ jobs:
               mkdir -p "$DAGGER_TMP_BINDIR"
               curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --release-stage=beta --release-stage=generally_available
+          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --release-stage=alpha
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index 4354f9e121db..f1b69f251c14 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -47,11 +47,12 @@ async def run(context: ConnectorTestContext) -> ConnectorTestReport:
         ConnectorTestReport: The test reports holding tests results.
     """
     async with context:
+        semaphore = anyio.Semaphore(2)
         async with asyncer.create_task_group() as task_group:
             tasks = [
                 task_group.soonify(checks.QaChecks(context).run)(),
                 task_group.soonify(checks.CodeFormatChecks(context).run)(),
-                task_group.soonify(tests.run_all_tests)(context),
+                task_group.soonify(tests.run_all_tests)(context, semaphore),
             ]
         results = list(itertools.chain(*(task.value for task in tasks)))
 
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
index 3161996c861f..f3f0bcd0698a 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
@@ -185,21 +185,22 @@ async def run(self, connector_under_test: Container) -> StepResult:
         return await self._run_tests_in_directory(connector_under_test_with_secrets, "integration_tests")
 
 
-async def run_all_tests(context: ConnectorTestContext) -> List[StepResult]:
-    package_install_results, connector_under_test = await ConnectorInstallTest(context).run()
-    unit_tests_results = await UnitTests(context).run(connector_under_test)
-    results = [
-        package_install_results,
-        unit_tests_results,
-    ]
-    if unit_tests_results.status is not StepStatus.SUCCESS:
-        return results + [IntegrationTests(context).skip(), AcceptanceTests(context).skip()]
-
-    context.secrets_dir = await secrets.get_connector_secret_dir(context)
-    async with asyncer.create_task_group() as task_group:
-        tasks = [
-            task_group.soonify(IntegrationTests(context).run)(connector_under_test),
-            task_group.soonify(AcceptanceTests(context).run)(),
+async def run_all_tests(context: ConnectorTestContext, semaphore) -> List[StepResult]:
+    async with semaphore:
+        package_install_results, connector_under_test = await ConnectorInstallTest(context).run()
+        unit_tests_results = await UnitTests(context).run(connector_under_test)
+        results = [
+            package_install_results,
+            unit_tests_results,
         ]
+        if unit_tests_results.status is not StepStatus.SUCCESS:
+            return results + [IntegrationTests(context).skip(), AcceptanceTests(context).skip()]
 
-    return results + [task.value for task in tasks]
+        context.secrets_dir = await secrets.get_connector_secret_dir(context)
+        async with asyncer.create_task_group() as task_group:
+            tasks = [
+                task_group.soonify(IntegrationTests(context).run)(connector_under_test),
+                task_group.soonify(AcceptanceTests(context).run)(),
+            ]
+
+        return results + [task.value for task in tasks]

From 539d23715cabe7b739172d1a99e835c46439637d Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Tue, 21 Mar 2023 23:48:56 +0100
Subject: [PATCH 04/18] disable qa check code format for testing semaphore

---
 .../ci_connector_ops/pipelines/connectors_ci.py           | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index f1b69f251c14..498e8c058589 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -50,8 +50,8 @@ async def run(context: ConnectorTestContext) -> ConnectorTestReport:
         semaphore = anyio.Semaphore(2)
         async with asyncer.create_task_group() as task_group:
             tasks = [
-                task_group.soonify(checks.QaChecks(context).run)(),
-                task_group.soonify(checks.CodeFormatChecks(context).run)(),
+                #task_group.soonify(checks.QaChecks(context).run)(),
+                #task_group.soonify(checks.CodeFormatChecks(context).run)(),
                 task_group.soonify(tests.run_all_tests)(context, semaphore),
             ]
         results = list(itertools.chain(*(task.value for task in tasks)))
@@ -194,7 +194,7 @@ def test_connectors(ctx: click.Context, names: Tuple[str], languages: Tuple[Conn
             ctx.obj["gha_workflow_run_url"],
             GITHUB_GLOBAL_DESCRIPTION,
             GITHUB_GLOBAL_CONTEXT,
-            should_send=ctx["ci_context"] == "pull_request",
+            should_send=ctx.obj.get("ci_context") == "pull_request",
             logger=logger,
         )
     except dagger.DaggerError as e:
@@ -205,7 +205,7 @@ def test_connectors(ctx: click.Context, names: Tuple[str], languages: Tuple[Conn
             ctx.obj["gha_workflow_run_url"],
             GITHUB_GLOBAL_DESCRIPTION,
             GITHUB_GLOBAL_CONTEXT,
-            should_send=ctx["ci_context"] == "pull_request",
+            should_send=ctx.obj.get("ci_context") == "pull_request",
             logger=logger,
         )
 

From 54f56527be63073c7974a086fa703ad42fd89cbe Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Wed, 22 Mar 2023 01:22:53 +0100
Subject: [PATCH 05/18] fix typo and change semaphore

---
 .../pipelines/connectors_ci.py                | 27 +++++++-------
 .../ci_connector_ops/pipelines/contexts.py    |  1 -
 .../ci_connector_ops/pipelines/tests.py       | 35 +++++++++----------
 3 files changed, 31 insertions(+), 32 deletions(-)

diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index 498e8c058589..2df4bbb08a84 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -36,7 +36,7 @@
 logger = logging.getLogger(__name__)
 
 
-async def run(context: ConnectorTestContext) -> ConnectorTestReport:
+async def run(context: ConnectorTestContext, semaphore: anyio.Semaphore) -> ConnectorTestReport:
     """Runs a CI pipeline for a single connector.
     A visual DAG can be found on the README.md file of the pipelines modules.
 
@@ -46,17 +46,17 @@ async def run(context: ConnectorTestContext) -> ConnectorTestReport:
     Returns:
         ConnectorTestReport: The test reports holding tests results.
     """
-    async with context:
-        semaphore = anyio.Semaphore(2)
-        async with asyncer.create_task_group() as task_group:
-            tasks = [
-                #task_group.soonify(checks.QaChecks(context).run)(),
-                #task_group.soonify(checks.CodeFormatChecks(context).run)(),
-                task_group.soonify(tests.run_all_tests)(context, semaphore),
-            ]
-        results = list(itertools.chain(*(task.value for task in tasks)))
-
-        context.test_report = ConnectorTestReport(context, steps_results=results)
+    async with semaphore:
+        async with context:
+            async with asyncer.create_task_group() as task_group:
+                tasks = [
+                    task_group.soonify(checks.QaChecks(context).run)(),
+                    task_group.soonify(checks.CodeFormatChecks(context).run)(),
+                    task_group.soonify(tests.run_all_tests)(context),
+                ]
+            results = list(itertools.chain(*(task.value for task in tasks)))
+
+            context.test_report = ConnectorTestReport(context, steps_results=results)
 
     return context.test_report
 
@@ -67,11 +67,12 @@ async def run_connectors_test_pipelines(contexts: List[ConnectorTestContext]):
     Args:
         contexts (List[ConnectorTestContext]): List of connector test contexts for which a CI pipeline needs to be run.
     """
+    semaphore = anyio.Semaphore(5)
     async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
         async with anyio.create_task_group() as tg:
             for context in contexts:
                 context.dagger_client = dagger_client.pipeline(f"{context.connector.technical_name} - Test Pipeline")
-                tg.start_soon(run, context)
+                tg.start_soon(run, context, semaphore)
 
 
 @click.group()
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py
index 4180cc018ae0..b7aec1416795 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py
@@ -177,6 +177,5 @@ async def __aexit__(self, exception_type, exception_value, traceback) -> bool:
                 )
                 if report_upload_exit_code != 0:
                     self.logger.error("Uploading the report to S3 failed.")
-
         await asyncify(update_commit_status_check)(**self.github_commit_status)
         return True
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
index f3f0bcd0698a..73b268787280 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
@@ -85,7 +85,7 @@ async def run(self) -> StepResult:
             StepResult: Failure or success of the acceptances tests with stdout and stdout.
         """
         if not self.context.connector.acceptance_test_config:
-            return StepResult(Step.ACCEPTANCE_TESTS, StepStatus.SKIPPED), None
+            return StepResult(self, StepStatus.SKIPPED), None
 
         dagger_client = self.get_dagger_pipeline(self.context.dagger_client)
 
@@ -185,22 +185,21 @@ async def run(self, connector_under_test: Container) -> StepResult:
         return await self._run_tests_in_directory(connector_under_test_with_secrets, "integration_tests")
 
 
-async def run_all_tests(context: ConnectorTestContext, semaphore) -> List[StepResult]:
-    async with semaphore:
-        package_install_results, connector_under_test = await ConnectorInstallTest(context).run()
-        unit_tests_results = await UnitTests(context).run(connector_under_test)
-        results = [
-            package_install_results,
-            unit_tests_results,
+async def run_all_tests(context: ConnectorTestContext) -> List[StepResult]:
+    package_install_results, connector_under_test = await ConnectorInstallTest(context).run()
+    unit_tests_results = await UnitTests(context).run(connector_under_test)
+    results = [
+        package_install_results,
+        unit_tests_results,
+    ]
+    if unit_tests_results.status is not StepStatus.SUCCESS:
+        return results + [IntegrationTests(context).skip(), AcceptanceTests(context).skip()]
+
+    context.secrets_dir = await secrets.get_connector_secret_dir(context)
+    async with asyncer.create_task_group() as task_group:
+        tasks = [
+            task_group.soonify(IntegrationTests(context).run)(connector_under_test),
+            task_group.soonify(AcceptanceTests(context).run)(),
         ]
-        if unit_tests_results.status is not StepStatus.SUCCESS:
-            return results + [IntegrationTests(context).skip(), AcceptanceTests(context).skip()]
 
-        context.secrets_dir = await secrets.get_connector_secret_dir(context)
-        async with asyncer.create_task_group() as task_group:
-            tasks = [
-                task_group.soonify(IntegrationTests(context).run)(connector_under_test),
-                task_group.soonify(AcceptanceTests(context).run)(),
-            ]
-
-        return results + [task.value for task in tasks]
+    return results + [task.value for task in tasks]

From c6aebd1eb814ae43b34bc6a264e248e6ee6d4ca9 Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Wed, 22 Mar 2023 09:55:10 +0100
Subject: [PATCH 06/18] disable acceptance tests

---
 .../pipelines/connectors_ci.py                | 26 +++++++++----------
 .../ci_connector_ops/pipelines/tests.py       |  3 ++-
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index 2df4bbb08a84..499a8a83a573 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -36,7 +36,7 @@
 logger = logging.getLogger(__name__)
 
 
-async def run(context: ConnectorTestContext, semaphore: anyio.Semaphore) -> ConnectorTestReport:
+async def run(context: ConnectorTestContext) -> ConnectorTestReport:
     """Runs a CI pipeline for a single connector.
     A visual DAG can be found on the README.md file of the pipelines modules.
 
@@ -46,17 +46,16 @@ async def run(context: ConnectorTestContext, semaphore: anyio.Semaphore) -> Conn
     Returns:
         ConnectorTestReport: The test reports holding tests results.
     """
-    async with semaphore:
-        async with context:
-            async with asyncer.create_task_group() as task_group:
-                tasks = [
-                    task_group.soonify(checks.QaChecks(context).run)(),
-                    task_group.soonify(checks.CodeFormatChecks(context).run)(),
-                    task_group.soonify(tests.run_all_tests)(context),
-                ]
-            results = list(itertools.chain(*(task.value for task in tasks)))
-
-            context.test_report = ConnectorTestReport(context, steps_results=results)
+    async with context:
+        async with asyncer.create_task_group() as task_group:
+            tasks = [
+                task_group.soonify(checks.QaChecks(context).run)(),
+                task_group.soonify(checks.CodeFormatChecks(context).run)(),
+                task_group.soonify(tests.run_all_tests)(context),
+            ]
+        results = list(itertools.chain(*(task.value for task in tasks)))
+
+        context.test_report = ConnectorTestReport(context, steps_results=results)
 
     return context.test_report
 
@@ -67,12 +66,11 @@ async def run_connectors_test_pipelines(contexts: List[ConnectorTestContext]):
     Args:
         contexts (List[ConnectorTestContext]): List of connector test contexts for which a CI pipeline needs to be run.
     """
-    semaphore = anyio.Semaphore(5)
     async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
         async with anyio.create_task_group() as tg:
             for context in contexts:
                 context.dagger_client = dagger_client.pipeline(f"{context.connector.technical_name} - Test Pipeline")
-                tg.start_soon(run, context, semaphore)
+                tg.start_soon(run, context)
 
 
 @click.group()
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
index 73b268787280..30ed75c457ed 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
@@ -96,6 +96,7 @@ async def run(self) -> StepResult:
 
         dockerd = (
             dagger_client.container()
+            .pipeline("DOCKERD")
             .from_("docker:23.0.1-dind")
             .with_mounted_cache("/var/lib/docker", dagger_client.cache_volume("docker-lib"), sharing=CacheSharingMode.PRIVATE)
             .with_mounted_cache("/tmp", dagger_client.cache_volume("share-tmp"))
@@ -199,7 +200,7 @@ async def run_all_tests(context: ConnectorTestContext) -> List[StepResult]:
     async with asyncer.create_task_group() as task_group:
         tasks = [
             task_group.soonify(IntegrationTests(context).run)(connector_under_test),
-            task_group.soonify(AcceptanceTests(context).run)(),
+            # task_group.soonify(AcceptanceTests(context).run)(),
         ]
 
     return results + [task.value for task in tasks]

From de31e56a1f68db9b9ffe34d872740959ac5244b7 Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Wed, 22 Mar 2023 10:01:50 +0100
Subject: [PATCH 07/18] use semaphore

---
 .../pipelines/connectors_ci.py                | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index 499a8a83a573..21749f9a17c6 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -36,7 +36,7 @@
 logger = logging.getLogger(__name__)
 
 
-async def run(context: ConnectorTestContext) -> ConnectorTestReport:
+async def run(context: ConnectorTestContext, semaphore: anyio.Semaphore) -> ConnectorTestReport:
     """Runs a CI pipeline for a single connector.
     A visual DAG can be found on the README.md file of the pipelines modules.
 
@@ -46,18 +46,19 @@ async def run(context: ConnectorTestContext) -> ConnectorTestReport:
     Returns:
         ConnectorTestReport: The test reports holding tests results.
     """
-    async with context:
-        async with asyncer.create_task_group() as task_group:
-            tasks = [
-                task_group.soonify(checks.QaChecks(context).run)(),
-                task_group.soonify(checks.CodeFormatChecks(context).run)(),
-                task_group.soonify(tests.run_all_tests)(context),
-            ]
-        results = list(itertools.chain(*(task.value for task in tasks)))
+    async with semaphore:
+        async with context:
+            async with asyncer.create_task_group() as task_group:
+                tasks = [
+                    task_group.soonify(checks.QaChecks(context).run)(),
+                    task_group.soonify(checks.CodeFormatChecks(context).run)(),
+                    task_group.soonify(tests.run_all_tests)(context),
+                ]
+            results = list(itertools.chain(*(task.value for task in tasks)))
 
-        context.test_report = ConnectorTestReport(context, steps_results=results)
+            context.test_report = ConnectorTestReport(context, steps_results=results)
 
-    return context.test_report
+        return context.test_report
 
 
 async def run_connectors_test_pipelines(contexts: List[ConnectorTestContext]):
@@ -66,11 +67,12 @@ async def run_connectors_test_pipelines(contexts: List[ConnectorTestContext]):
     Args:
         contexts (List[ConnectorTestContext]): List of connector test contexts for which a CI pipeline needs to be run.
     """
+    semaphore = anyio.Semaphore(3)
     async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
         async with anyio.create_task_group() as tg:
             for context in contexts:
                 context.dagger_client = dagger_client.pipeline(f"{context.connector.technical_name} - Test Pipeline")
-                tg.start_soon(run, context)
+                tg.start_soon(run, context, semaphore)
 
 
 @click.group()

From 4a2a2a377db039b21261831191cd885bf55acd3a Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Wed, 22 Mar 2023 10:26:45 +0100
Subject: [PATCH 08/18] restore ga builds

---
 .github/workflows/connector_nightly_builds_dagger.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index 430ad5e62fa2..73593054f119 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -41,7 +41,7 @@ jobs:
               mkdir -p "$DAGGER_TMP_BINDIR"
               curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --release-stage=alpha
+          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --release-stage=generally_available --release-stage=beta
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}

From cef82ad66daf97143359f59ee74b41545e2785fb Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 08:24:22 +0100
Subject: [PATCH 09/18] use CacheSharingMode.SHARED, renable acceptance tests,
 semaphore of 5

---
 .../ci_connector_ops/pipelines/actions/environments.py          | 2 +-
 .../ci_connector_ops/pipelines/connectors_ci.py                 | 2 +-
 tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
index eac3f373ceb7..353f3ab8e777 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
@@ -50,7 +50,7 @@ def with_python_base(context: ConnectorTestContext, python_image_name: str = "py
     return (
         context.dagger_client.container()
         .from_(python_image_name)
-        .with_mounted_cache("/root/.cache/pip", pip_cache, sharing=CacheSharingMode.LOCKED)
+        .with_mounted_cache("/root/.cache/pip", pip_cache, sharing=CacheSharingMode.SHARED)
         .with_mounted_directory("/tools", context.get_repo_dir("tools", include=["ci_credentials", "ci_common_utils"], exclude=[".venv"]))
         .with_exec(["pip", "install", "--upgrade", "pip"])
     )
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index 21749f9a17c6..86497c11b87d 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -67,7 +67,7 @@ async def run_connectors_test_pipelines(contexts: List[ConnectorTestContext]):
     Args:
         contexts (List[ConnectorTestContext]): List of connector test contexts for which a CI pipeline needs to be run.
     """
-    semaphore = anyio.Semaphore(3)
+    semaphore = anyio.Semaphore(5)
     async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
         async with anyio.create_task_group() as tg:
             for context in contexts:
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
index 30ed75c457ed..8e6a3acdc9e3 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
@@ -200,7 +200,7 @@ async def run_all_tests(context: ConnectorTestContext) -> List[StepResult]:
     async with asyncer.create_task_group() as task_group:
         tasks = [
             task_group.soonify(IntegrationTests(context).run)(connector_under_test),
-            # task_group.soonify(AcceptanceTests(context).run)(),
+            task_group.soonify(AcceptanceTests(context).run)(),
         ]
 
     return results + [task.value for task in tasks]

From 111efb020077c9ca84a65ae1f752ea513ef44ede Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 12:10:12 +0100
Subject: [PATCH 10/18] expose concurrency as a CLI option, set it to 10 on the
 workflow

---
 .../workflows/connector_nightly_builds_dagger.yml    |  2 +-
 .../ci_connector_ops/pipelines/connectors_ci.py      | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index 73593054f119..e9dc5fd8a84f 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -41,7 +41,7 @@ jobs:
               mkdir -p "$DAGGER_TMP_BINDIR"
               curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --release-stage=generally_available --release-stage=beta
+          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --concurrency=10 --release-stage=generally_available --release-stage=beta
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
index 86497c11b87d..7c6efc059f4d 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/connectors_ci.py
@@ -61,13 +61,14 @@ async def run(context: ConnectorTestContext, semaphore: anyio.Semaphore) -> Conn
         return context.test_report
 
 
-async def run_connectors_test_pipelines(contexts: List[ConnectorTestContext]):
+async def run_connectors_test_pipelines(contexts: List[ConnectorTestContext], concurrency: int = 5):
     """Runs a CI pipeline for all the connectors passed.
 
     Args:
         contexts (List[ConnectorTestContext]): List of connector test contexts for which a CI pipeline needs to be run.
+        concurrency (int): Number of test pipeline that can run in parallel. Defaults to 5
     """
-    semaphore = anyio.Semaphore(5)
+    semaphore = anyio.Semaphore(concurrency)
     async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
         async with anyio.create_task_group() as tg:
             for context in contexts:
@@ -142,8 +143,11 @@ def connectors_ci(
     type=click.Choice(["alpha", "beta", "generally_available"]),
 )
 @click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool)
+@click.option("--concurrency", help="Number of connector tests pipeline to run in parallel.", default=5, type=int)
 @click.pass_context
-def test_connectors(ctx: click.Context, names: Tuple[str], languages: Tuple[ConnectorLanguage], release_stages: Tuple[str], modified: bool):
+def test_connectors(
+    ctx: click.Context, names: Tuple[str], languages: Tuple[ConnectorLanguage], release_stages: Tuple[str], modified: bool, concurrency: int
+):
     """Runs a CI pipeline the connector passed as CLI argument.
 
     Args:
@@ -188,7 +192,7 @@ def test_connectors(ctx: click.Context, names: Tuple[str], languages: Tuple[Conn
         in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]  # TODO: remove this once we implement pipelines for Java connector
     ]
     try:
-        anyio.run(run_connectors_test_pipelines, connectors_tests_contexts)
+        anyio.run(run_connectors_test_pipelines, connectors_tests_contexts, concurrency)
         update_commit_status_check(
             ctx.obj["git_revision"],
             "success",

From 4bf38019e61b5b669e588440cd49c031f910b442 Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 18:14:55 +0100
Subject: [PATCH 11/18] clean

---
 tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
index 8e6a3acdc9e3..73b268787280 100644
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
+++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests.py
@@ -96,7 +96,6 @@ async def run(self) -> StepResult:
 
         dockerd = (
             dagger_client.container()
-            .pipeline("DOCKERD")
             .from_("docker:23.0.1-dind")
             .with_mounted_cache("/var/lib/docker", dagger_client.cache_volume("docker-lib"), sharing=CacheSharingMode.PRIVATE)
             .with_mounted_cache("/tmp", dagger_client.cache_volume("share-tmp"))

From 9bffa2146196f4df2ca4a22393a99370ef238fde Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 18:21:49 +0100
Subject: [PATCH 12/18] use xlarge runner

---
 .github/workflows/connector_nightly_builds_dagger.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index e9dc5fd8a84f..7a20dce65985 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -10,7 +10,7 @@ jobs:
   nightly_builds_alpha:
     name: Nightly builds GA + BETA
     timeout-minutes: 360 # 6 hours
-    runs-on: large-runner
+    runs-on: xlarge-runner
     steps:
       - name: Get start timestamp
         id: get-start-timestamp

From caed1ec7f29224daa021de8de3bfce99c1816dae Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 18:24:06 +0100
Subject: [PATCH 13/18] // 15

---
 .github/workflows/connector_nightly_builds_dagger.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index 7a20dce65985..b46c51805241 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -41,7 +41,7 @@ jobs:
               mkdir -p "$DAGGER_TMP_BINDIR"
               curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --concurrency=10 --release-stage=generally_available --release-stage=beta
+          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --concurrency=15 --release-stage=generally_available --release-stage=beta
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}

From 7ff7164adba5470fb6284481c0dd9f5219ffc383 Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 19:13:35 +0100
Subject: [PATCH 14/18] declare inputs

---
 .../workflows/connector_nightly_builds_dagger.yml    | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index b46c51805241..c5aa862aecf9 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -5,12 +5,20 @@ on:
     # 11AM UTC is 12AM CET, 4AM PST.
     - cron: "0 11 * * *"
   workflow_dispatch:
+    inputs:
+      runs-on:
+        type: string
+        default: large-runner
+        required: true
+      test-connectors-options:
+        default: --concurrency=5 --release-stage=generally_available --release-stage=beta
+        required: true
 
 jobs:
   nightly_builds_alpha:
     name: Nightly builds GA + BETA
     timeout-minutes: 360 # 6 hours
-    runs-on: xlarge-runner
+    runs-on: ${{ inputs.runs-on || 'large-runner' }}
     steps:
       - name: Get start timestamp
         id: get-start-timestamp
@@ -41,7 +49,7 @@ jobs:
               mkdir -p "$DAGGER_TMP_BINDIR"
               curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors --concurrency=15 --release-stage=generally_available --release-stage=beta
+          connectors-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} test-connectors ${{ inputs.test-connectors-options || '--concurrency=5 --release-stage=generally_available --release-stage=beta' }}
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}

From 74023a176fa1e5669de0493b533f0acf22e653ee Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 19:19:16 +0100
Subject: [PATCH 15/18] better job naming

---
 .github/workflows/connector_nightly_builds_dagger.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index c5aa862aecf9..0c0c99b594dc 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -15,8 +15,8 @@ on:
         required: true
 
 jobs:
-  nightly_builds_alpha:
-    name: Nightly builds GA + BETA
+  test_connectors:
+    name: Test connectors ${{ test-connectors-options }}
     timeout-minutes: 360 # 6 hours
     runs-on: ${{ inputs.runs-on || 'large-runner' }}
     steps:
@@ -39,7 +39,7 @@ jobs:
           python-version: "3.10"
       - name: Install ci-connector-ops package
         run: pip install ./tools/ci_connector_ops\[pipelines]\
-      - name: Run nightly builds
+      - name: Test connectors
         run: |
           export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
           DAGGER_CLI_COMMIT="67c7e7635cf4ea0e446e2fed522a3e314c960f6a"

From aeea5f6ce43d4b6ea8e2913714124e796c1a9fa0 Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 19:20:29 +0100
Subject: [PATCH 16/18] better job naming

---
 .github/workflows/connector_nightly_builds_dagger.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index 0c0c99b594dc..69a6a6377d7e 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -16,7 +16,7 @@ on:
 
 jobs:
   test_connectors:
-    name: Test connectors ${{ test-connectors-options }}
+    name: Test connectors ${{ inputs.test-connectors-options }}
     timeout-minutes: 360 # 6 hours
     runs-on: ${{ inputs.runs-on || 'large-runner' }}
     steps:

From 98455bc0e2cb36dd55cb168bc8d94ca07f71f2bc Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 19:23:19 +0100
Subject: [PATCH 17/18] use run-name

---
 .github/workflows/connector_nightly_builds_dagger.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index 69a6a6377d7e..9315f1b36860 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -14,6 +14,8 @@ on:
         default: --concurrency=5 --release-stage=generally_available --release-stage=beta
         required: true
 
+run-name: Test connectors ${{ inputs.test-connectors-options }}
+
 jobs:
   test_connectors:
     name: Test connectors ${{ inputs.test-connectors-options }}

From de3cf17d624cda5792bfab78ce0143a07408fb4b Mon Sep 17 00:00:00 2001
From: alafanechere <augustin.lafanechere@gmail.com>
Date: Thu, 23 Mar 2023 19:24:36 +0100
Subject: [PATCH 18/18] use run-name

---
 .github/workflows/connector_nightly_builds_dagger.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connector_nightly_builds_dagger.yml
index 9315f1b36860..db8e9c55dd25 100644
--- a/.github/workflows/connector_nightly_builds_dagger.yml
+++ b/.github/workflows/connector_nightly_builds_dagger.yml
@@ -14,11 +14,11 @@ on:
         default: --concurrency=5 --release-stage=generally_available --release-stage=beta
         required: true
 
-run-name: Test connectors ${{ inputs.test-connectors-options }}
+run-name: Test connectors ${{ inputs.test-connectors-options || 'Nightly builds GA and Beta connectors' }}
 
 jobs:
   test_connectors:
-    name: Test connectors ${{ inputs.test-connectors-options }}
+    name: Test connectors ${{ inputs.test-connectors-options || 'Nightly builds GA and Beta connectors' }}
     timeout-minutes: 360 # 6 hours
     runs-on: ${{ inputs.runs-on || 'large-runner' }}
     steps: