From 5298498525876a03c5cec3b6a32d4b16be6a0f83 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 31 Jan 2023 16:03:53 -0800 Subject: [PATCH 01/11] Checkout entire git history --- .github/workflows/test-command.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 43d120122626..b1466c7af31b 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -82,6 +82,7 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + fetch-depth: 0 - name: Install Java uses: actions/setup-java@v3 with: From 9521cecab09464425d95956a864de910d6840d30 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Feb 2023 13:20:56 -0800 Subject: [PATCH 02/11] Debug directory --- tools/ci_connector_ops/ci_connector_ops/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index c2b367ce27ac..e119088f5dab 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -16,7 +16,13 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) os.chdir('../../..') -AIRBYTE_REPO = git.Repo(".") +# print the current working directory +print("Current working directory:") +print(os.getcwd()) + +REPO_ROOT = "." + +AIRBYTE_REPO = git.Repo(REPO_ROOT) DIFFED_BRANCH = "origin/master" OSS_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/oss_catalog.json" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" From 6852d021a0568fa6e54069d7dcc340c25a039c14 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Feb 2023 13:22:28 -0800 Subject: [PATCH 03/11] Use relative path --- tools/ci_connector_ops/ci_connector_ops/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index e119088f5dab..d76cbe29489d 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -13,15 +13,14 @@ import yaml # ensure we are at the repository root -os.chdir(os.path.dirname(os.path.abspath(__file__))) -os.chdir('../../..') +# os.chdir(os.path.dirname(os.path.abspath(__file__))) +# os.chdir('../../..') # print the current working directory print("Current working directory:") print(os.getcwd()) -REPO_ROOT = "." - +REPO_ROOT = "../../" AIRBYTE_REPO = git.Repo(REPO_ROOT) DIFFED_BRANCH = "origin/master" OSS_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/oss_catalog.json" From d4da790eed3b74befe722494303b742aa2bb4a2a Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Feb 2023 19:45:26 -0800 Subject: [PATCH 04/11] Test for path --- tools/ci_connector_ops/ci_connector_ops/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index d76cbe29489d..3a12bf197013 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -13,15 +13,14 @@ import yaml # ensure we are at the repository root -# os.chdir(os.path.dirname(os.path.abspath(__file__))) -# os.chdir('../../..') +os.chdir(os.path.dirname(os.path.abspath(__file__))) +os.chdir('../../..') # print the current working directory print("Current working directory:") print(os.getcwd()) -REPO_ROOT = "../../" -AIRBYTE_REPO = git.Repo(REPO_ROOT) +AIRBYTE_REPO = git.Repo(".") DIFFED_BRANCH = "origin/master" OSS_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/oss_catalog.json" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" From 8e7891502b9feb5fdbeb0345f07abf6dcab4ad78 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Feb 2023 11:28:18 -0800 Subject: [PATCH 05/11] Search parent directories --- tools/ci_connector_ops/ci_connector_ops/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index 3a12bf197013..36ca624f141f 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -20,7 +20,12 @@ print("Current working directory:") print(os.getcwd()) -AIRBYTE_REPO = git.Repo(".") +AIRBYTE_REPO = git.Repo(search_parent_directories=True) + +branch = AIRBYTE_REPO.active_branch +print(f"Active Branch: {branch.name}") + + DIFFED_BRANCH = "origin/master" OSS_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/oss_catalog.json" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" From d496d77647825207130cfaa88beb479e5ca3b5f6 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Feb 2023 12:01:50 -0800 Subject: [PATCH 06/11] Set current director to file --- tools/ci_connector_ops/ci_connector_ops/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index 36ca624f141f..f39aaffc1041 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -14,13 +14,13 @@ # ensure we are at the repository root os.chdir(os.path.dirname(os.path.abspath(__file__))) -os.chdir('../../..') +# os.chdir('../../..') # print the current working directory print("Current working directory:") print(os.getcwd()) -AIRBYTE_REPO = git.Repo(search_parent_directories=True) +AIRBYTE_REPO = git.Repo(".", search_parent_directories=True) branch = AIRBYTE_REPO.active_branch print(f"Active Branch: {branch.name}") From e48d6328e83739920aeb9dbffcb8260d857f08dd Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Feb 2023 12:34:20 -0800 Subject: [PATCH 07/11] Add a few more debug statements --- tools/ci_connector_ops/ci_connector_ops/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index f39aaffc1041..dde3b3ac8fa3 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -12,13 +12,19 @@ import requests import yaml +file_path = os.path.abspath(__file__) + +print(f"Start directory: {os.getcwd()}") +print(f"File: {__file__}") +print(f"File path: {file_path}") +print(f"File directory: {os.path.dirname(file_path)}") + # ensure we are at the repository root os.chdir(os.path.dirname(os.path.abspath(__file__))) # os.chdir('../../..') # print the current working directory -print("Current working directory:") -print(os.getcwd()) +print(f"Current working directory: {os.getcwd()}") AIRBYTE_REPO = git.Repo(".", search_parent_directories=True) From 624bc23d23618bcd9c117031259bd69a51cb0ce2 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Feb 2023 13:15:01 -0800 Subject: [PATCH 08/11] Remove change dir --- tools/ci_connector_ops/ci_connector_ops/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index dde3b3ac8fa3..b68cb86e86c9 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -20,13 +20,13 @@ print(f"File directory: {os.path.dirname(file_path)}") # ensure we are at the repository root -os.chdir(os.path.dirname(os.path.abspath(__file__))) +# os.chdir(os.path.dirname(os.path.abspath(__file__))) # os.chdir('../../..') # print the current working directory print(f"Current working directory: {os.getcwd()}") -AIRBYTE_REPO = git.Repo(".", search_parent_directories=True) +AIRBYTE_REPO = git.Repo(search_parent_directories=True) branch = AIRBYTE_REPO.active_branch print(f"Active Branch: {branch.name}") From 7e429dac146672270254396099fb9e36c46d3519 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Feb 2023 13:34:27 -0800 Subject: [PATCH 09/11] Reenable qa-checks on master --- .github/workflows/test-command.yml | 3 +-- .../ci_connector_ops/utils.py | 25 ------------------- tools/status/report.sh | 5 +--- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index b1466c7af31b..22803b51e219 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -153,8 +153,7 @@ jobs: retention-days: 3 - name: Run QA checks for ${{ github.event.inputs.connector }} id: qa_checks - # TODO: Disabled for on master until #22127 resolved - if: contains(github.ref, 'feat-qa-engine') + if: always() run: | run-qa-checks ${{ github.event.inputs.connector }} - name: Report Status diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index b68cb86e86c9..473baf2784e8 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -2,36 +2,15 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # - from dataclasses import dataclass import logging from pathlib import Path from typing import Dict, Optional, Set, Tuple, List -import os import git import requests import yaml -file_path = os.path.abspath(__file__) - -print(f"Start directory: {os.getcwd()}") -print(f"File: {__file__}") -print(f"File path: {file_path}") -print(f"File directory: {os.path.dirname(file_path)}") - -# ensure we are at the repository root -# os.chdir(os.path.dirname(os.path.abspath(__file__))) -# os.chdir('../../..') - -# print the current working directory -print(f"Current working directory: {os.getcwd()}") - AIRBYTE_REPO = git.Repo(search_parent_directories=True) - -branch = AIRBYTE_REPO.active_branch -print(f"Active Branch: {branch.name}") - - DIFFED_BRANCH = "origin/master" OSS_CATALOG_URL = "https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/oss_catalog.json" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" @@ -42,15 +21,12 @@ DESTINATION_DEFINITIONS_FILE_PATH = "airbyte-config/init/src/main/resources/seed/destination_definitions.yaml" DEFINITIONS_FILE_PATH = {"source": SOURCE_DEFINITIONS_FILE_PATH, "destination": DESTINATION_DEFINITIONS_FILE_PATH} - def download_catalog(catalog_url): response = requests.get(catalog_url) return response.json() - OSS_CATALOG = download_catalog(OSS_CATALOG_URL) - class ConnectorInvalidNameError(Exception): pass @@ -64,7 +40,6 @@ def read_definitions(definitions_file_path: str) -> Dict: def get_connector_name_from_path(path): return path.split("/")[2] - def get_changed_acceptance_test_config(diff_regex: Optional[str]=None) -> Set[str]: """Retrieve a list of connector names for which the acceptance_test_config file was changed in the current branch (compared to master). diff --git a/tools/status/report.sh b/tools/status/report.sh index b14f1194038e..5af90f378f88 100755 --- a/tools/status/report.sh +++ b/tools/status/report.sh @@ -15,10 +15,7 @@ CONNECTOR=$1 REPOSITORY=$2 RUN_ID=$3 TEST_OUTCOME=$4 - -# TODO: Disabled for on master until #22127 resolved -# QA_CHECKS_OUTCOME=$5 -QA_CHECKS_OUTCOME=success +QA_CHECKS_OUTCOME=$5 # Ensure connector is prefixed with connectors/ # TODO (ben): In the future we should just hard error if this is not the case From 285db31ae84e10b64b61e60c1dddece1b9583abb Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Feb 2023 13:36:03 -0800 Subject: [PATCH 10/11] Remove fetch depth and retest --- .github/workflows/test-command.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 22803b51e219..43d120122626 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -82,7 +82,6 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} - fetch-depth: 0 - name: Install Java uses: actions/setup-java@v3 with: @@ -153,7 +152,8 @@ jobs: retention-days: 3 - name: Run QA checks for ${{ github.event.inputs.connector }} id: qa_checks - if: always() + # TODO: Disabled for on master until #22127 resolved + if: contains(github.ref, 'feat-qa-engine') run: | run-qa-checks ${{ github.event.inputs.connector }} - name: Report Status From b7e67d052387ab05baa572d971d7a5b106df0f3b Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Feb 2023 14:00:53 -0800 Subject: [PATCH 11/11] Enable qa-checks --- .github/workflows/test-command.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 43d120122626..11f2214d9b83 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -152,8 +152,7 @@ jobs: retention-days: 3 - name: Run QA checks for ${{ github.event.inputs.connector }} id: qa_checks - # TODO: Disabled for on master until #22127 resolved - if: contains(github.ref, 'feat-qa-engine') + if: always() run: | run-qa-checks ${{ github.event.inputs.connector }} - name: Report Status