diff --git a/airbyte-integrations/connectors/destination-redshift-py/.dockerignore b/airbyte-integrations/connectors/destination-redshift-no-dbt/.dockerignore similarity index 53% rename from airbyte-integrations/connectors/destination-redshift-py/.dockerignore rename to airbyte-integrations/connectors/destination-redshift-no-dbt/.dockerignore index e0bd879b7900..28a23da0399b 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/.dockerignore +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/.dockerignore @@ -1,5 +1,5 @@ * !Dockerfile !main.py -!destination_redshift_py +!destination_redshift_no_dbt !setup.py diff --git a/airbyte-integrations/connectors/destination-redshift-py/Dockerfile b/airbyte-integrations/connectors/destination-redshift-no-dbt/Dockerfile similarity index 90% rename from airbyte-integrations/connectors/destination-redshift-py/Dockerfile rename to airbyte-integrations/connectors/destination-redshift-no-dbt/Dockerfile index 741adbc58899..abd4bfb21e41 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/Dockerfile @@ -31,10 +31,10 @@ RUN apk --no-cache add bash \ # copy payload code only COPY main.py ./ -COPY destination_redshift_py ./destination_redshift_py +COPY destination_redshift_no_dbt ./destination_redshift_no_dbt ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-redshift-py +LABEL io.airbyte.name=airbyte/destination-redshift-no-dbt diff --git a/airbyte-integrations/connectors/destination-redshift-py/README.md b/airbyte-integrations/connectors/destination-redshift-no-dbt/README.md similarity index 88% rename from airbyte-integrations/connectors/destination-redshift-py/README.md rename to airbyte-integrations/connectors/destination-redshift-no-dbt/README.md index 61c6be4f8245..6f2715280736 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/README.md +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/README.md @@ -1,7 +1,7 @@ -# Redshift Py Destination +# Redshift No Dbt Destination -This is the repository for the Redshift Py destination connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/redshift-py). +This is the repository for the Redshift No Dbt destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/redshift-no-dbt). ## Local development @@ -32,16 +32,16 @@ should work as you expect. #### Building via Gradle From the Airbyte repository root, run: ``` -./gradlew :airbyte-integrations:connectors:destination-redshift-py:build +./gradlew :airbyte-integrations:connectors:destination-redshift-no-dbt:build ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/redshift-py) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_redshift_py/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/redshift-no-dbt) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_redshift_no_dbt/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination redshift-py test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination redshift-no-dbt test creds` and place them into `secrets/config.json`. ### Locally running the connector @@ -57,12 +57,12 @@ python main.py read --config secrets/config.json --catalog integration_tests/con #### Build First, make sure you build the latest Docker image: ``` -docker build . -t airbyte/destination-redshift-py:dev +docker build . -t airbyte/destination-redshift-no-dbt:dev ``` You can also build the connector image via Gradle: ``` -./gradlew :airbyte-integrations:connectors:destination-redshift-py:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-redshift-no-dbt:airbyteDocker ``` When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in the Dockerfile. @@ -70,10 +70,10 @@ the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/destination-redshift-py:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-redshift-py:dev check --config /secrets/config.json +docker run --rm airbyte/destination-redshift-no-dbt:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-redshift-no-dbt:dev check --config /secrets/config.json # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages -cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-redshift-py:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-redshift-no-dbt:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. @@ -101,11 +101,11 @@ Coming soon: All commands should be run from airbyte project root. To run unit tests: ``` -./gradlew :airbyte-integrations:connectors:destination-redshift-py:unitTest +./gradlew :airbyte-integrations:connectors:destination-redshift-no-dbt:unitTest ``` To run acceptance and custom integration tests: ``` -./gradlew :airbyte-integrations:connectors:destination-redshift-py:integrationTest +./gradlew :airbyte-integrations:connectors:destination-redshift-no-dbt:integrationTest ``` ## Dependency Management diff --git a/airbyte-integrations/connectors/destination-redshift-py/build.gradle b/airbyte-integrations/connectors/destination-redshift-no-dbt/build.gradle similarity index 61% rename from airbyte-integrations/connectors/destination-redshift-py/build.gradle rename to airbyte-integrations/connectors/destination-redshift-no-dbt/build.gradle index 34b3a1d6c43b..3a7a35ae4f9b 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/build.gradle @@ -4,5 +4,5 @@ plugins { } airbytePython { - moduleDirectory 'destination_redshift_py' + moduleDirectory 'destination_redshift_no_dbt' } diff --git a/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/__init__.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/__init__.py new file mode 100644 index 000000000000..257fd1b486b6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationRedshiftNoDbt + +__all__ = ["DestinationRedshiftNoDbt"] diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/csv_writer.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/csv_writer.py similarity index 96% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/csv_writer.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/csv_writer.py index 9f4ecf418786..35cc001caa5f 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/csv_writer.py +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/csv_writer.py @@ -7,7 +7,7 @@ from dotmap import DotMap -from destination_redshift_py.table import Table +from destination_redshift_no_dbt.table import Table CSV_EXTENSION = "csv" GZIP_EXTENSION = "gzip" diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/data_type_converter.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/data_type_converter.py similarity index 96% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/data_type_converter.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/data_type_converter.py index 2d7a04bcfcb1..e37a44618154 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/data_type_converter.py +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/data_type_converter.py @@ -1,6 +1,6 @@ from typing import List, Union -from destination_redshift_py.field import DataType +from destination_redshift_no_dbt.field import DataType VARCHAR = "VARCHAR" TIMESTAMP_WITHOUT_TIME_ZONE = "TIMESTAMP WITHOUT TIME ZONE" diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/destination.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/destination.py similarity index 96% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/destination.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/destination.py index 81503e0a1767..bb6fdb9c4dd5 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/destination.py +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/destination.py @@ -18,16 +18,16 @@ from psycopg2._psycopg import connection as Connection from psycopg2.pool import ThreadedConnectionPool, SimpleConnectionPool -from destination_redshift_py.csv_writer import CSVWriter -from destination_redshift_py.jsonschema_to_tables import JsonToTables, PARENT_CHILD_SPLITTER -from destination_redshift_py.s3_objects_manager import S3ObjectsManager -from destination_redshift_py.stream import Stream -from destination_redshift_py.table import AIRBYTE_EMITTED_AT, AIRBYTE_ID_NAME, Table +from destination_redshift_no_dbt.csv_writer import CSVWriter +from destination_redshift_no_dbt.jsonschema_to_tables import JsonToTables, PARENT_CHILD_SPLITTER +from destination_redshift_no_dbt.s3_objects_manager import S3ObjectsManager +from destination_redshift_no_dbt.stream import Stream +from destination_redshift_no_dbt.table import AIRBYTE_EMITTED_AT, AIRBYTE_ID_NAME, Table logger = logging.getLogger("airbyte") -class DestinationRedshiftPy(Destination): +class DestinationRedshiftNoDbt(Destination): def __init__(self): self.streams: Dict[str, Stream] = dict() @@ -157,7 +157,7 @@ def get_records(parents: Union[DotMap, List[DotMap]], method: str) -> List[DotMa @staticmethod def _assign_id_and_emitted_at(records: List[DotMap], emitted_at: int, hashing_keys: List[str]): for record in records: - DestinationRedshiftPy._assign_id(record=record, hashing_keys=hashing_keys) + DestinationRedshiftNoDbt._assign_id(record=record, hashing_keys=hashing_keys) record[AIRBYTE_EMITTED_AT.name] = datetime.utcfromtimestamp(emitted_at / 1000).isoformat(timespec="seconds") @staticmethod diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/field.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/field.py similarity index 100% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/field.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/field.py diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/jsonschema_to_tables.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/jsonschema_to_tables.py similarity index 92% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/jsonschema_to_tables.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/jsonschema_to_tables.py index 934b382d81af..6c738af2c6f5 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/jsonschema_to_tables.py +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/jsonschema_to_tables.py @@ -1,9 +1,9 @@ from collections import OrderedDict from typing import Optional, List -from destination_redshift_py.data_type_converter import DataTypeConverter, FALLBACK_DATATYPE -from destination_redshift_py.field import Field -from destination_redshift_py.table import Table +from destination_redshift_no_dbt.data_type_converter import DataTypeConverter, FALLBACK_DATATYPE +from destination_redshift_no_dbt.field import Field +from destination_redshift_no_dbt.table import Table PARENT_CHILD_SPLITTER = "." diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/s3_objects_manager.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/s3_objects_manager.py similarity index 100% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/s3_objects_manager.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/s3_objects_manager.py diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/spec.json b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/spec.json similarity index 99% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/spec.json rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/spec.json index 5ba050fb4127..8732b363ef6d 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/spec.json +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift-py", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/redshift-no-dbt", "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], "supportsIncremental": true, "supportsDBT": false, diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/stream.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/stream.py similarity index 86% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/stream.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/stream.py index ff70b5541a00..92d4e7a36b18 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/stream.py +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/stream.py @@ -1,7 +1,7 @@ from typing import Dict from airbyte_cdk.models import DestinationSyncMode -from destination_redshift_py.table import Table +from destination_redshift_no_dbt.table import Table from pydantic.main import BaseModel diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/table.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/table.py similarity index 95% rename from airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/table.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/table.py index 601663e22a1e..2aeffb100ab6 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/table.py +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/destination_redshift_no_dbt/table.py @@ -1,7 +1,7 @@ from typing import List, Optional -from destination_redshift_py.data_type_converter import VARCHAR, TIMESTAMP_WITHOUT_TIME_ZONE -from destination_redshift_py.field import Field, DataType +from destination_redshift_no_dbt.data_type_converter import VARCHAR, TIMESTAMP_WITHOUT_TIME_ZONE +from destination_redshift_no_dbt.field import Field, DataType AIRBYTE_ID_NAME = "_airbyte_ab_id" AIRBYTE_EMITTED_AT_NAME = "_airbyte_emitted_at" diff --git a/airbyte-integrations/connectors/destination-redshift-py/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/integration_tests/integration_test.py similarity index 100% rename from airbyte-integrations/connectors/destination-redshift-py/integration_tests/integration_test.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/integration_tests/integration_test.py diff --git a/airbyte-integrations/connectors/destination-redshift-no-dbt/main.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/main.py new file mode 100644 index 000000000000..f169393a0c3b --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_redshift_no_dbt import DestinationRedshiftNoDbt + +if __name__ == "__main__": + DestinationRedshiftNoDbt().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-redshift-py/requirements.txt b/airbyte-integrations/connectors/destination-redshift-no-dbt/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/destination-redshift-py/requirements.txt rename to airbyte-integrations/connectors/destination-redshift-no-dbt/requirements.txt diff --git a/airbyte-integrations/connectors/destination-redshift-py/sample_files/configured_catalog.json b/airbyte-integrations/connectors/destination-redshift-no-dbt/sample_files/configured_catalog.json similarity index 100% rename from airbyte-integrations/connectors/destination-redshift-py/sample_files/configured_catalog.json rename to airbyte-integrations/connectors/destination-redshift-no-dbt/sample_files/configured_catalog.json diff --git a/airbyte-integrations/connectors/destination-redshift-py/setup.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/setup.py similarity index 82% rename from airbyte-integrations/connectors/destination-redshift-py/setup.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/setup.py index 5f8ca4e84bd5..6247c77870ce 100644 --- a/airbyte-integrations/connectors/destination-redshift-py/setup.py +++ b/airbyte-integrations/connectors/destination-redshift-no-dbt/setup.py @@ -14,8 +14,8 @@ ] setup( - name="destination_redshift_py", - description="Destination implementation for Redshift Py.", + name="destination_redshift_no_dbt", + description="Destination implementation for Redshift No Dbt.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), diff --git a/airbyte-integrations/connectors/destination-redshift-py/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-redshift-no-dbt/unit_tests/unit_test.py similarity index 100% rename from airbyte-integrations/connectors/destination-redshift-py/unit_tests/unit_test.py rename to airbyte-integrations/connectors/destination-redshift-no-dbt/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/__init__.py b/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/__init__.py deleted file mode 100644 index 93ad23186301..000000000000 --- a/airbyte-integrations/connectors/destination-redshift-py/destination_redshift_py/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from .destination import DestinationRedshiftPy - -__all__ = ["DestinationRedshiftPy"] diff --git a/airbyte-integrations/connectors/destination-redshift-py/main.py b/airbyte-integrations/connectors/destination-redshift-py/main.py deleted file mode 100644 index 63fb60abf449..000000000000 --- a/airbyte-integrations/connectors/destination-redshift-py/main.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -import sys - -from destination_redshift_py import DestinationRedshiftPy - -if __name__ == "__main__": - DestinationRedshiftPy().run(sys.argv[1:])