airbytehq · alafanechere · Dec 6, 2023 · Nov 28, 2023 · Dec 6, 2023
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,66 +1,11 @@
-default_language_version:
-  python: python3
-
 repos:
-  - repo: https://github.com/johann-petrak/licenseheaders.git
-    rev: v0.8.8
-    hooks:
-      - id: licenseheaders
-        args:
-          ["--tmpl=LICENSE_SHORT", "--ext=py", "-x=**/models/__init__.py", "-f"]
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-      - id: black
-        args: ["--config", "pyproject.toml"]
-  - repo: https://github.com/timothycrosley/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        args:
-          [
-            "--settings-file",
-            "pyproject.toml",
-            "--dont-follow-links",
-            "--jobs=-1",
-          ]
-        additional_dependencies: ["colorama"]
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.5.0
-    hooks:
-      - id: prettier
-        types_or: [yaml, json]
-        exclude: |
-          (?x)^.*(
-              .github/|
-              .gitlab-ci.yml
-          ).?$
-
-  - repo: https://github.com/csachs/pyproject-flake8
-    rev: v6.0.0
-    hooks:
-      - id: pyproject-flake8
-        args: ["--config", "pyproject.toml"]
-        additional_dependencies: ["mccabe"]
-        alias: flake8
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.930
-    hooks:
-      - id: mypy
-        args: ["--config-file", "pyproject.toml"]
-        exclude: |
-          (?x)^.*(
-              octavia-cli/unit_tests/|
-          ).?$
   - repo: local
     hooks:
-      - id: spec-linter
-        name: validate connectors spec files
+      - id: format-fix-all-on-push
+        always_run: true
+        entry: airbyte-ci format fix all
         language: system
-        entry: python tools/git_hooks/spec_linter.py
-        files: ^.*/spec.json$
-        exclude: |
-          (?x)^.*(
-              /connectors/destination-e2e-test|
-              /connectors/source-e2e-test
-          ).*$
+        name: Run airbyte-ci format fix on git push
+        pass_filenames: false
+        stages: [push]
+        verbose: true
diff --git a/Makefile b/Makefile
@@ -15,7 +15,14 @@ tools.airbyte-ci.check: # Check if airbyte-ci is installed correctly
 tools.airbyte-ci.clean: ## Clean airbyte-ci installations
 	@./airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_clean.sh
 
-tools.install: tools.airbyte-ci.install tools.airbyte-ci.check
 
-.PHONY: tools.install tools.airbyte-ci.install tools.airbyte-ci-dev.install tools.airbyte-ci.check tools.airbyte-ci.clean
+tools.pre-commit.install:
+	@echo "Installing pre-commit..."
+	@brew install pre-commit
+	@echo "Installing pre-push hooks..."
+	@pre-commit install --hook-type pre-push
+	@echo "Pre-commit installation complete and pre-push hooks installed."
+
+tools.install: tools.airbyte-ci.install tools.airbyte-ci.check tools.pre-commit.install
 
+.PHONY: tools.install tools.pre-commit.install tools.airbyte-ci.install tools.airbyte-ci-dev.install tools.airbyte-ci.check tools.airbyte-ci.clean
diff --git a/airbyte-cdk/python/airbyte_cdk/test/http/__init__.py b/airbyte-cdk/python/airbyte_cdk/test/http/__init__.py
@@ -1,6 +1,9 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 from airbyte_cdk.test.http.matcher import HttpRequestMatcher
+from airbyte_cdk.test.http.mocker import HttpMocker
 from airbyte_cdk.test.http.request import HttpRequest
 from airbyte_cdk.test.http.response import HttpResponse
-from airbyte_cdk.test.http.mocker import HttpMocker
 
 __all__ = ["HttpMocker", "HttpRequest", "HttpRequestMatcher", "HttpResponse"]
diff --git a/airbyte-cdk/python/airbyte_cdk/test/http/matcher.py b/airbyte-cdk/python/airbyte_cdk/test/http/matcher.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 from airbyte_cdk.test.http.request import HttpRequest
 

diff --git a/airbyte-cdk/python/airbyte_cdk/test/http/mocker.py b/airbyte-cdk/python/airbyte_cdk/test/http/mocker.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 import contextlib
 import functools

diff --git a/airbyte-cdk/python/airbyte_cdk/test/http/request.py b/airbyte-cdk/python/airbyte_cdk/test/http/request.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 from typing import Any, List, Mapping, Optional, Union
 from urllib.parse import parse_qs, urlencode, urlparse

diff --git a/airbyte-cdk/python/airbyte_cdk/test/http/response.py b/airbyte-cdk/python/airbyte_cdk/test/http/response.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 
 class HttpResponse:

diff --git a/airbyte-cdk/python/airbyte_cdk/test/http/response_builder.py b/airbyte-cdk/python/airbyte_cdk/test/http/response_builder.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 import functools
 import json
@@ -117,10 +119,7 @@ def build(self) -> Dict[str, Any]:
 
 class HttpResponseBuilder:
     def __init__(
-        self,
-        template: Dict[str, Any],
-        records_path: Union[FieldPath, NestedPath],
-        pagination_strategy: Optional[PaginationStrategy]
+        self, template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], pagination_strategy: Optional[PaginationStrategy]
     ):
         self._response = template
         self._records: List[RecordBuilder] = []
@@ -170,7 +169,7 @@ def create_builders_from_resource(
     records_path: Union[FieldPath, NestedPath],
     record_id_path: Optional[Path] = None,
     record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None,
-    pagination_strategy: Optional[PaginationStrategy] = None
+    pagination_strategy: Optional[PaginationStrategy] = None,
 ) -> Tuple[RecordBuilder, HttpResponseBuilder]:
     """
     This will use the first record define at `records_path` as a template for the records. If more records are defined, they will be ignored
@@ -181,11 +180,13 @@ def create_builders_from_resource(
     try:
         record_template = records_path.extract(response_template)[0]
         if not record_template:
-            raise ValueError(f"Could not extract any record from template at path `{records_path}`. "
-                             f"Please fix the template to provide a record sample or fix `records_path`.")
+            raise ValueError(
+                f"Could not extract any record from template at path `{records_path}`. "
+                f"Please fix the template to provide a record sample or fix `records_path`."
+            )
         return (
             RecordBuilder(record_template, record_id_path, record_cursor_path),
-            HttpResponseBuilder(response_template, records_path, pagination_strategy)
+            HttpResponseBuilder(response_template, records_path, pagination_strategy),
         )
     except (IndexError, KeyError):
         raise ValueError(f"Error while extracting records at path `{records_path}` from response template `{response_template}`")
diff --git a/airbyte-cdk/python/unit_tests/test/__init__.py b/airbyte-cdk/python/unit_tests/test/__init__.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
diff --git a/airbyte-cdk/python/unit_tests/test/http/__init__.py b/airbyte-cdk/python/unit_tests/test/http/__init__.py
@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
diff --git a/airbyte-cdk/python/unit_tests/test/http/test_matcher.py b/airbyte-cdk/python/unit_tests/test/http/test_matcher.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 from unittest import TestCase
 from unittest.mock import Mock

diff --git a/airbyte-cdk/python/unit_tests/test/http/test_mocker.py b/airbyte-cdk/python/unit_tests/test/http/test_mocker.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 from unittest import TestCase
 

diff --git a/airbyte-cdk/python/unit_tests/test/http/test_request.py b/airbyte-cdk/python/unit_tests/test/http/test_request.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 
 from unittest import TestCase
 

diff --git a/airbyte-cdk/python/unit_tests/test/http/test_response_builder.py b/airbyte-cdk/python/unit_tests/test/http/test_response_builder.py
@@ -1,4 +1,6 @@
+#
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
 import json
 from copy import deepcopy
 from pathlib import Path as FilePath
@@ -41,9 +43,7 @@ def _record_builder(
 
 
 def _response_builder(
-    response_template: Dict[str, Any],
-    records_path: Union[FieldPath, NestedPath],
-    pagination_strategy: Optional[PaginationStrategy] = None
+    response_template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], pagination_strategy: Optional[PaginationStrategy] = None
 ) -> HttpResponseBuilder:
     return create_builders_from_resource(deepcopy(response_template), records_path, pagination_strategy=pagination_strategy)[1]
 
@@ -59,7 +59,9 @@ def test_given_with_id_when_build_then_set_id(self) -> None:
         assert record[_ID_FIELD] == "another id"
 
     def test_given_nested_id_when_build_then_set_id(self) -> None:
-        builder = _record_builder({_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD]))
+        builder = _record_builder(
+            {_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD])
+        )
         record = builder.with_id("another id").build()
         assert record["nested"][_ID_FIELD] == "another id"
 
@@ -74,9 +76,7 @@ def test_given_no_id_in_template_for_path_when_build_then_raise_error(self) -> N
 
     def test_given_with_cursor_when_build_then_set_id(self) -> None:
         builder = _record_builder(
-            {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]},
-            FieldPath(_RECORDS_FIELD),
-            record_cursor_path=FieldPath(_CURSOR_FIELD)
+            {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]}, FieldPath(_RECORDS_FIELD), record_cursor_path=FieldPath(_CURSOR_FIELD)
         )
         record = builder.with_cursor("another cursor").build()
         assert record[_CURSOR_FIELD] == "another cursor"
@@ -85,7 +85,7 @@ def test_given_nested_cursor_when_build_then_set_cursor(self) -> None:
         builder = _record_builder(
             {_RECORDS_FIELD: [{"nested": {_CURSOR_FIELD: "a cursor"}}]},
             FieldPath(_RECORDS_FIELD),
-            record_cursor_path=NestedPath(["nested", _CURSOR_FIELD])
+            record_cursor_path=NestedPath(["nested", _CURSOR_FIELD]),
         )
         record = builder.with_cursor("another cursor").build()
         assert record["nested"][_CURSOR_FIELD] == "another cursor"
@@ -100,7 +100,7 @@ def test_given_no_cursor_in_template_for_path_when_build_then_raise_error(self)
             _record_builder(
                 {_RECORDS_FIELD: [{"record without cursor": "should fail"}]},
                 FieldPath(_RECORDS_FIELD),
-                record_cursor_path=FieldPath(_ID_FIELD)
+                record_cursor_path=FieldPath(_ID_FIELD),
             )
 
 
@@ -135,7 +135,7 @@ def test_given_pagination_with_strategy_when_build_then_apply_strategy(self) ->
         builder = _response_builder(
             {"has_more_pages": False} | _SOME_RECORDS,
             FieldPath(_RECORDS_FIELD),
-            pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page")
+            pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page"),
         )
 
         response = builder.with_pagination().build()

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py
@@ -31,7 +31,7 @@ async def test(pipeline_context: ClickPipelineContext):
 
     # The following directories are always mounted because a lot of tests rely on them
     directories_to_always_mount = [
-        ".git", # This is needed as some package tests rely on being in a git repo
+        ".git",  # This is needed as some package tests rely on being in a git repo
         ".github",
         "docs",
         "airbyte-integrations",

diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/java.java b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/java.java
@@ -1,8 +1,14 @@
+/*
+ * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+ */
+
 public class BadlyFormatted {
-public static void main(String[] args) {
-System.out.println("Hello, World!");
-for (int i=0; i<5; i++){
-System.out.println(i);
+
+  public static void main(String[] args) {
+    System.out.println("Hello, World!");
+    for (int i = 0; i < 5; i++) {
+      System.out.println(i);
+    }
+  }
+
 }
-}
-}
diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/python.py b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/python.py
@@ -1,4 +1,9 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+
 def my_function():
-    print('Using single quotes, no newlines, and no spaces between the function name.')
+    print("Using single quotes, no newlines, and no spaces between the function name.")
+
+
 def my_other_function():
-    print('Using single quotes, no newlines, and no spaces between the function name.')
+    print("Using single quotes, no newlines, and no spaces between the function name.")
diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/yaml.yaml b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/yaml.yaml
@@ -1,4 +1,4 @@
 name: John Doe
 age: 25
 city: XYZ
-street: 123 Main St
+street: 123 Main St
diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py
@@ -84,57 +84,34 @@ def test_discovery_uniquely_named_streams():
                 "$schema": "https://json-schema.org/draft-07/schema#",
                 "type": ["null", "object"],
                 "properties": {
-                    "amount": {
-                        "type": ["null", "integer"]
-                    },
-                    "amount_details": {
-                        "type": ["null", "object"],
-                        "properties": {
-                            "atm_fee": ["null", "integer"]
-                        }
-                    }
-                }
+                    "amount": {"type": ["null", "integer"]},
+                    "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": ["null", "integer"]}},
+                },
             },
-            True
+            True,
         ),
         (
             {
                 "$schema": "https://json-schema.org/draft-07/schema#",
                 "type": ["null", "object"],
                 "properties": {
                     "amount": "integer",
-                    "amount_details": {
-                        "type": ["null", "object"],
-                        "properties": {
-                            "atm_fee": {
-                                "type": ["null", "integer"]
-                            }
-                        }
-                    }
-                }
+                    "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}},
+                },
             },
-            True
+            True,
         ),
         (
             {
                 "$schema": "https://json-schema.org/draft-07/schema#",
                 "type": ["null", "object"],
                 "properties": {
-                    "amount": {
-                        "type": ["null", "integer"]
-                    },
-                    "amount_details": {
-                        "type": ["null", "object"],
-                        "properties": {
-                            "atm_fee": {
-                                "type": ["null", "integer"]
-                            }
-                        }
-                    }
-                }
+                    "amount": {"type": ["null", "integer"]},
+                    "amount_details": {"type": ["null", "object"], "properties": {"atm_fee": {"type": ["null", "integer"]}}},
+                },
             },
-            False
-        )
+            False,
+        ),
     ],
 )
 def test_streams_have_valid_json_schemas(schema, should_fail):