connectors-ci: fix pip install error (#29156)

airbyte-ci/connectors/pipelines/README.md

@@ -377,25 +377,25 @@ This command runs the Python tests for a airbyte-ci poetry package.
 `airbyte-ci tests connectors/pipelines`
 
 ## Changelog
-
-| Version | PR                                                        | Description                                                                                  |
-|---------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------|
-| 0.4.6   | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional  paramater in Dagger's API      |
-| 0.4.5   | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish.                                             |
-| 0.4.4   | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set.                                                  |
-| 0.4.3   | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors.                                             |
-| 0.4.2   | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`.                                 |
-| 0.4.1   | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands.                           |
-| 0.4.0   | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI                                                             |
-| 0.3.2   | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull.                                                |
-| 0.3.1   | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped                                            |
-| 0.3.0   | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution                                |
-| 0.2.3   | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command                                    |
-| 0.2.2   | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting                                  |
-| 0.2.1   | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive.                    |
-| 0.2.0   | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. |
-| 0.1.1   | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn.                              |
-| 0.1.0   |                                                           | Alpha version not in production yet. All the commands described in this doc are available.   |
+| Version | PR                                                        | Description                                                                                               |
+| ------- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 0.4.7   | [#29156](https://github.com/airbytehq/airbyte/pull/29156) | Improve how we check existence of requirement.txt or setup.py file to not raise early pip install errors. |
+| 0.4.6   | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional  paramater in Dagger's API                   |
+| 0.4.5   | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish.                                                          |
+| 0.4.4   | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set.                                                               |
+| 0.4.3   | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors.                                                          |
+| 0.4.2   | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`.                                              |
+| 0.4.1   | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands.                                        |
+| 0.4.0   | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI                                                                          |
+| 0.3.2   | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull.                                                             |
+| 0.3.1   | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped                                                         |
+| 0.3.0   | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution                                             |
+| 0.2.3   | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command                                                 |
+| 0.2.2   | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting                                               |
+| 0.2.1   | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive.                                 |
+| 0.2.0   | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package.              |
+| 0.1.1   | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn.                                           |
+| 0.1.0   |                                                           | Alpha version not in production yet. All the commands described in this doc are available.                |
 
 ## More info
 This project is owned by the Connectors Operations team.

airbyte-ci/connectors/pipelines/pipelines/actions/environments.py

@@ -24,7 +24,7 @@
     LICENSE_SHORT_FILE_PATH,
     PYPROJECT_TOML_FILE_PATH,
 )
-from pipelines.utils import get_file_contents
+from pipelines.utils import check_path_in_workdir, get_file_contents
 
 if TYPE_CHECKING:
     from pipelines.contexts import ConnectorContext, PipelineContext
@@ -312,9 +312,13 @@ async def with_installed_python_package(
     for dependency_directory in local_dependencies:
         container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory))
 
-    if await get_file_contents(container, "setup.py"):
+    has_setup_py, has_requirements_txt = await check_path_in_workdir(container, "setup.py"), await check_path_in_workdir(
+        container, "requirements.txt"
+    )
+
+    if has_setup_py:
         container = container.with_exec(install_connector_package_cmd)
-    if await get_file_contents(container, "requirements.txt"):
+    if has_requirements_txt:
         container = container.with_exec(install_requirements_cmd)
 
     if additional_dependency_groups:

airbyte-ci/connectors/pipelines/pipelines/bases.py

@@ -187,7 +187,7 @@ def should_retry(self, step_result: StepResult) -> bool:
     async def retry(self, step_result, *args, **kwargs) -> StepResult:
         self.retry_count += 1
         self.logger.warn(
-            f"Failed with error: {step_result.stderr}. Retry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..."
+            f"Failed with error: {step_result.stderr}.\nRetry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..."
         )
         await anyio.sleep(self.retry_delay.total_seconds())
         return await self.run(*args, **kwargs)

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "0.4.6"
+version = "0.4.7"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py

@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import pytest
+from connector_ops.utils import Connector
+from pipelines.actions import environments
+from pipelines.contexts import PipelineContext
+
+pytestmark = [
+    pytest.mark.anyio,
+]
+
+
+@pytest.fixture
+def python_connector() -> Connector:
+    return Connector("source-openweather")
+
+
+@pytest.fixture
+def context(dagger_client):
+    context = PipelineContext(
+        pipeline_name="test",
+        is_local=True,
+        git_branch="test",
+        git_revision="test",
+    )
+    context.dagger_client = dagger_client
+    return context
+
+
+async def test_with_installed_python_package(context, python_connector):
+    python_environment = context.dagger_client.container().from_("python:3.9")
+    installed_connector_package = await environments.with_installed_python_package(
+        context,
+        python_environment,
+        str(python_connector.code_directory),
+    )
+    await installed_connector_package.with_exec(["python", "main.py", "spec"])

airbyte-ci/connectors/pipelines/tests/test_tests/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

airbyte-ci/connectors/pipelines/tests/test_utils.py

@@ -5,7 +5,7 @@
 from unittest import mock
 
 import pytest
-from connector_ops.utils import ConnectorLanguage
+from connector_ops.utils import Connector, ConnectorLanguage
 from pipelines import utils
 from tests.utils import pick_a_random_connector
 
@@ -165,3 +165,18 @@ def test_no_modified_files_in_connector_directory():
 
     result = utils.get_connector_modified_files(connector, all_modified_files)
     assert result == frozenset()
+
+
+@pytest.mark.anyio
+async def test_check_path_in_workdir(dagger_client):
+    connector = Connector("source-openweather")
+    container = (
+        dagger_client.container()
+        .from_("bash")
+        .with_mounted_directory(str(connector.code_directory), dagger_client.host().directory(str(connector.code_directory)))
+        .with_workdir(str(connector.code_directory))
+    )
+    assert await utils.check_path_in_workdir(container, "metadata.yaml")
+    assert await utils.check_path_in_workdir(container, "setup.py")
+    assert await utils.check_path_in_workdir(container, "requirements.txt")
+    assert await utils.check_path_in_workdir(container, "not_existing_file") is False