airbytehq · alafanechere · Oct 14, 2024 · Oct 8, 2024 · Oct 9, 2024 · Oct 11, 2024
@@ -37,6 +37,7 @@ poetry run pytest
 ```
 
 ## Changelog
+- 0.10.0: Add `documentation_file_name` property to `Connector` class.
 - 0.9.0: Add components path attribute for manifest-only connectors.
 - 0.8.1: Gradle dependency discovery logic supports the Bulk CDK.
 - 0.8.0: Add a `sbom_url` property to `Connector`

@@ -301,6 +301,10 @@ def relative_documentation_path_str(self) -> str:
 
         return f"./docs/{relative_documentation_path}"
 
+    @property
+    def documentation_file_name(self) -> str:
+        return self.metadata.get("documentationUrl").split("/")[-1] + ".md"
+
     @property
     def documentation_file_path(self) -> Optional[Path]:
         return Path(f"{self.relative_documentation_path_str}.md") if self.has_airbyte_docs else None

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "connector_ops"
-version = "0.9.0"
+version = "0.10.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors"
 authors = ["Airbyte <contact@airbyte.io>"]
 
@@ -22,6 +22,7 @@ google-cloud-storage = "^2.8.0"
 ci-credentials = {path = "../ci_credentials"}
 pandas = "^2.0.3"
 simpleeval = "^0.9.13"
+semver = "^3.0.2"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8"

@@ -6,6 +6,7 @@
 from pathlib import Path
 
 import pytest
+import semver
 from connector_ops import utils
 
 
@@ -45,7 +46,7 @@ def test_init(self, connector, exists, mocker, tmp_path):
             assert isinstance(connector.support_level, str)
             assert isinstance(connector.acceptance_test_config, dict)
             assert connector.icon_path == Path(f"./airbyte-integrations/connectors/{connector.technical_name}/icon.svg")
-            assert len(connector.version.split(".")) == 3
+            assert semver.Version.parse(connector.version)
         else:
             assert connector.metadata is None
             assert connector.support_level is None

@@ -76,7 +76,8 @@ async def run_connector_version_bump_pipeline(
     async with semaphore:
         steps_results = []
         async with context:
-            bump_version = BumpConnectorVersion(context, bump_type)
+            connector_directory = await context.get_connector_dir()
+            bump_version = BumpConnectorVersion(context, connector_directory, bump_type)
             bump_version_result = await bump_version.run()
             steps_results.append(bump_version_result)
             if not bump_version_result.success:
@@ -88,9 +89,9 @@ async def run_connector_version_bump_pipeline(
             for modified_file in bump_version.modified_files:
                 await updated_connector_directory.file(modified_file).export(str(context.connector.code_directory / modified_file))
                 context.logger.info(f"Exported {modified_file} following the version bump.")
-
+            documentation_directory = await context.get_repo_dir(include=[str(context.connector.local_connector_documentation_directory)])
             add_changelog_entry = AddChangelogEntry(
-                context, bump_version.new_version, changelog_entry, pull_request_number=pull_request_number
+                context, documentation_directory, bump_version.new_version, changelog_entry, pull_request_number=pull_request_number
             )
             add_changelog_entry_result = await add_changelog_entry.run()
             steps_results.append(add_changelog_entry_result)

@@ -1,7 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-from typing import List, Optional
+from typing import List
 
 import asyncclick as click
 from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID

@@ -193,6 +193,7 @@ async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, s
             og_repo_dir = await context.get_repo_dir()
             update_base_image_in_metadata = UpdateBaseImageMetadata(
                 context,
+                await context.get_connector_dir(),
                 set_if_not_exists=set_if_not_exists,
             )
             update_base_image_in_metadata_result = await update_base_image_in_metadata.run()
@@ -232,6 +233,7 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte
             # UPDATE BASE IMAGE IN METADATA
             update_base_image_in_metadata = UpdateBaseImageMetadata(
                 context,
+                await context.get_connector_dir(),
                 set_if_not_exists=True,
             )
             update_base_image_in_metadata_result = await update_base_image_in_metadata.run()

@@ -18,7 +18,6 @@
 from pipelines.airbyte_ci.connectors.reports import Report
 from pipelines.consts import LOCAL_BUILD_PLATFORM
 from pipelines.helpers.connectors.command import run_connector_steps
-from pipelines.helpers.connectors.format import format_prettier
 from pipelines.helpers.connectors.yaml import read_yaml, write_yaml
 from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun
 from pipelines.models.steps import Step, StepResult, StepStatus

@@ -5,11 +5,57 @@
 import copy
 import logging
 import typing
-from typing import Any, Mapping, Optional, Set, Type
+from typing import Any, Dict, Mapping, Optional, Set, Type
 
 from pydantic import BaseModel
 
-from .declarative_component_schema import *
+from .declarative_component_schema import (
+    ApiKeyAuthenticator,
+    BasicHttpAuthenticator,
+    BearerAuthenticator,
+    CompositeErrorHandler,
+    ConstantBackoffStrategy,
+    CursorPagination,
+    CustomAuthenticator,
+    CustomBackoffStrategy,
+    CustomErrorHandler,
+    CustomIncrementalSync,
+    CustomPaginationStrategy,
+    CustomPartitionRouter,
+    CustomRecordExtractor,
+    CustomRecordFilter,
+    CustomRequester,
+    CustomRetriever,
+    CustomSchemaLoader,
+    CustomStateMigration,
+    CustomTransformation,
+    DatetimeBasedCursor,
+    DeclarativeSource,
+    DeclarativeStream,
+    DefaultErrorHandler,
+    DefaultPaginator,
+    DpathExtractor,
+    ExponentialBackoffStrategy,
+    HttpRequester,
+    HttpResponseFilter,
+    JsonFileSchemaLoader,
+    JwtAuthenticator,
+    LegacySessionTokenAuthenticator,
+    ListPartitionRouter,
+    MinMaxDatetime,
+    OAuthAuthenticator,
+    OffsetIncrement,
+    PageIncrement,
+    ParentStreamConfig,
+    RecordFilter,
+    RecordSelector,
+    SelectiveAuthenticator,
+    SessionTokenAuthenticator,
+    SimpleRetriever,
+    SubstreamPartitionRouter,
+    WaitTimeFromHeader,
+    WaitUntilTimeFromHeader,
+)
 
 PARAMETERS_STR = "$parameters"
 

@@ -101,7 +101,7 @@ def preprocess_manifest(self, manifest: Mapping[str, Any]) -> Mapping[str, Any]:
         """
         return self._evaluate_node(manifest, manifest, set())  # type: ignore[no-any-return]
 
-    def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set[Any]) -> Any:
+    def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set[Any]) -> Any:  # noqa: ANN401
         if isinstance(node, dict):
             evaluated_dict = {k: self._evaluate_node(v, manifest, visited) for k, v in node.items() if not self._is_ref_key(k)}
             if REF_TAG in node:
@@ -126,7 +126,7 @@ def _evaluate_node(self, node: Any, manifest: Mapping[str, Any], visited: Set[An
         else:
             return node
 
-    def _lookup_ref_value(self, ref: str, manifest: Mapping[str, Any]) -> Any:
+    def _lookup_ref_value(self, ref: str, manifest: Mapping[str, Any]) -> Any:  # noqa: ANN401
         ref_match = re.match(r"#/(.*)", ref)
         if not ref_match:
             raise ValueError(f"Invalid reference format {ref}")
@@ -137,15 +137,15 @@ def _lookup_ref_value(self, ref: str, manifest: Mapping[str, Any]) -> Any:
             raise ValueError(f"{path}, {ref}")
 
     @staticmethod
-    def _is_ref(node: Any) -> bool:
+    def _is_ref(node: Any) -> bool:  # noqa: ANN401
         return isinstance(node, str) and node.startswith("#/")
 
     @staticmethod
     def _is_ref_key(key: str) -> bool:
         return bool(key == REF_TAG)
 
     @staticmethod
-    def _read_ref_value(ref: str, manifest_node: Mapping[str, Any]) -> Any:
+    def _read_ref_value(ref: str, manifest_node: Mapping[str, Any]) -> Any:  # noqa: ANN401
         """
         Read the value at the referenced location of the manifest.
 

@@ -191,6 +191,7 @@ async def publish(
                 python_registry_url=python_registry_url,
                 python_registry_check_url=python_registry_check_url,
                 rollout_mode=rollout_mode,
+                ci_github_access_token=ctx.obj.get("ci_github_access_token"),
             )
             for connector in ctx.obj["selected_connectors_with_modified_files"]
         ]

@@ -55,6 +55,7 @@ def __init__(
         s3_build_cache_secret_key: Optional[Secret] = None,
         use_local_cdk: bool = False,
         python_registry_token: Optional[Secret] = None,
+        ci_github_access_token: Optional[Secret] = None,
     ) -> None:
         self.pre_release = pre_release
         self.spec_cache_bucket_name = spec_cache_bucket_name
@@ -94,6 +95,7 @@ def __init__(
             docker_hub_password=docker_hub_password,
             s3_build_cache_access_key_id=s3_build_cache_access_key_id,
             s3_build_cache_secret_key=s3_build_cache_secret_key,
+            ci_github_access_token=ci_github_access_token,
         )
 
         # Reassigning current class required instance attribute