From 93c0dbff4be173e1bf77c92a268e012e545cae21 Mon Sep 17 00:00:00 2001 From: nailo2c Date: Wed, 23 Apr 2025 21:30:03 -0700 Subject: [PATCH 1/6] add exasol operators doc, example and refactor trino --- providers/exasol/docs/index.rst | 7 ++ providers/exasol/docs/operators.rst | 79 ++++++++++++++++ providers/exasol/tests/system/__init__.py | 0 .../exasol/tests/system/exasol/__init__.py | 0 .../tests/system/exasol/example_exasol.py | 91 +++++++++++++++++++ .../{operators/transfer => }/gcs_to_trino.rst | 0 providers/trino/docs/index.rst | 4 +- .../{operators/trino.rst => operators.rst} | 0 providers/trino/docs/operators/index.rst | 25 ----- 9 files changed, 179 insertions(+), 27 deletions(-) create mode 100644 providers/exasol/docs/operators.rst create mode 100644 providers/exasol/tests/system/__init__.py create mode 100644 providers/exasol/tests/system/exasol/__init__.py create mode 100644 providers/exasol/tests/system/exasol/example_exasol.py rename providers/trino/docs/{operators/transfer => }/gcs_to_trino.rst (100%) rename providers/trino/docs/{operators/trino.rst => operators.rst} (100%) delete mode 100644 providers/trino/docs/operators/index.rst diff --git a/providers/exasol/docs/index.rst b/providers/exasol/docs/index.rst index d7755960f016b..0afa727b48387 100644 --- a/providers/exasol/docs/index.rst +++ b/providers/exasol/docs/index.rst @@ -29,6 +29,13 @@ Changelog Security +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Operators + .. toctree:: :hidden: :maxdepth: 1 diff --git a/providers/exasol/docs/operators.rst b/providers/exasol/docs/operators.rst new file mode 100644 index 0000000000000..4d8e6f2c0fb37 --- /dev/null +++ b/providers/exasol/docs/operators.rst @@ -0,0 +1,79 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/operator:ExasolOperator: + +SQLExecuteQueryOperator to connect to Exasol +==================================================== + +Use the :class:`SQLExecuteQueryOperator` to execute +SQL commands in an `Exasol `__ database. + +.. note:: + Previously, an ``ExasolOperator`` was used to perform this kind of operation. + After deprecation this has been removed. Please use ``SQLExecuteQueryOperator`` instead. + +.. note:: + Make sure you have installed the ``apache-airflow-providers-exasol`` package + and its dependency ``pyexasol`` to enable Exasol support. + +Using the Operator +^^^^^^^^^^^^^^^^^^ + +Use the ``conn_id`` argument to connect to your Exasol instance where +the connection metadata is structured as follows: + +.. list-table:: Exasol Airflow Connection Metadata + :widths: 25 25 + :header-rows: 1 + + * - Parameter + - Input + * - Host: string + - Exasol hostname, container alias or IP address + * - Schema: string + - Default schema name (e.g. ``TEST``) (optional) + * - Login: string + - Exasol username (e.g. ``TEST`` or ``sys``) + * - Password: string + - Exasol password + * - Port: int + - Exasol port (default: 8563) + * - Extra: JSON + - Additional connection configuration passed to *pyexasol*, such as: + ``{"encryption": false}`` or + ``{"encryption": true, "websocket_sslopt": {"cert_reqs": 0}}`` + +An example usage of the SQLExecuteQueryOperator to connect to Exasol is as follows: + +.. exampleinclude:: /../../exasol/tests/system/exasol/example_exasol.py + :language: python + :start-after: [START howto_operator_exasol] + :end-before: [END howto_operator_exasol] + +Reference +^^^^^^^^^ +For further information, look at: + +* `Exasol Documentation `__ + +.. note:: + + Parameters provided directly via SQLExecuteQueryOperator() take precedence + over those specified in the Airflow connection metadata (such as ``schema``, ``login``, ``password``, etc). diff --git a/providers/exasol/tests/system/__init__.py b/providers/exasol/tests/system/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/exasol/tests/system/exasol/__init__.py b/providers/exasol/tests/system/exasol/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/exasol/tests/system/exasol/example_exasol.py b/providers/exasol/tests/system/exasol/example_exasol.py new file mode 100644 index 0000000000000..d615733253a16 --- /dev/null +++ b/providers/exasol/tests/system/exasol/example_exasol.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This is an example DAG for the use of the SQLExecuteQueryOperator with Exasol. +""" + +from __future__ import annotations + +import datetime + +from airflow import DAG +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.providers.exasol.hooks.exasol import exasol_fetch_all_handler + +DAG_ID = "example_exasol" + +with DAG( + dag_id=DAG_ID, + start_date=datetime.datetime(2025, 1, 1), + default_args={"conn_id": "my_exasol_conn", "handler": exasol_fetch_all_handler}, + schedule="@once", + catchup=False, +) as dag: + # [START howto_operator_exasol] + create_table_exasol = SQLExecuteQueryOperator( + task_id="create_table_exasol", + sql=""" + CREATE OR REPLACE TABLE exasol_example ( + a VARCHAR(100), + b DECIMAL(18,0) + ); + """, + ) + + alter_table_exasol = SQLExecuteQueryOperator( + task_id="alter_table_exasol", + sql="ALTER TABLE exasol_example ADD COLUMN c DECIMAL(18,0);", + ) + + insert_data_exasol = SQLExecuteQueryOperator( + task_id="insert_data_exasol", + sql=""" + INSERT INTO exasol_example (a, b, c) + VALUES + ('a', 1, 1), + ('a', 2, 1), + ('b', 3, 1); + """, + ) + + select_data_exasol = SQLExecuteQueryOperator( + task_id="select_data_exasol", + sql="SELECT * FROM exasol_example;", + ) + + drop_table_exasol = SQLExecuteQueryOperator( + task_id="drop_table_exasol", + sql="DROP TABLE exasol_example;", + ) + # [END howto_operator_exasol] + + ( + create_table_exasol + >> alter_table_exasol + >> insert_data_exasol + >> select_data_exasol + >> drop_table_exasol + ) + + from tests_common.test_utils.watcher import watcher + + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest +test_run = get_test_run(dag) diff --git a/providers/trino/docs/operators/transfer/gcs_to_trino.rst b/providers/trino/docs/gcs_to_trino.rst similarity index 100% rename from providers/trino/docs/operators/transfer/gcs_to_trino.rst rename to providers/trino/docs/gcs_to_trino.rst diff --git a/providers/trino/docs/index.rst b/providers/trino/docs/index.rst index 30cb1263cb1cf..d4563b46220e7 100644 --- a/providers/trino/docs/index.rst +++ b/providers/trino/docs/index.rst @@ -34,8 +34,8 @@ :maxdepth: 1 :caption: Guides - TrinoTransferOperator types - TrinoOperator + TrinoTransferOperator types + Operators Connection types .. toctree:: diff --git a/providers/trino/docs/operators/trino.rst b/providers/trino/docs/operators.rst similarity index 100% rename from providers/trino/docs/operators/trino.rst rename to providers/trino/docs/operators.rst diff --git a/providers/trino/docs/operators/index.rst b/providers/trino/docs/operators/index.rst deleted file mode 100644 index d51bca0f7ae0a..0000000000000 --- a/providers/trino/docs/operators/index.rst +++ /dev/null @@ -1,25 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Trino operator -============== - -.. toctree:: - :maxdepth: 1 - :glob: - - * From 4d4ff0924eb646d126057dacc60388beaf5f2346 Mon Sep 17 00:00:00 2001 From: nailo2c Date: Wed, 23 Apr 2025 22:26:22 -0700 Subject: [PATCH 2/6] fix lint issue --- providers/exasol/docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/providers/exasol/docs/index.rst b/providers/exasol/docs/index.rst index 0afa727b48387..b495f10ec0cff 100644 --- a/providers/exasol/docs/index.rst +++ b/providers/exasol/docs/index.rst @@ -42,6 +42,7 @@ :caption: References Python API <_api/airflow/providers/exasol/index> + Tests API <_api/tests/system/exasol/index> .. toctree:: :hidden: From 2e7835facfb4499cb15a7b5bd1d261c5d0978859 Mon Sep 17 00:00:00 2001 From: nailo2c Date: Wed, 23 Apr 2025 22:59:58 -0700 Subject: [PATCH 3/6] fix lint issue #2 --- providers/exasol/docs/index.rst | 2 +- providers/exasol/tests/system/__init__.py | 16 ++++++++++++++++ providers/exasol/tests/system/exasol/__init__.py | 16 ++++++++++++++++ .../exasol/tests/system/exasol/example_exasol.py | 8 ++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/providers/exasol/docs/index.rst b/providers/exasol/docs/index.rst index b495f10ec0cff..7b21c901c2fcf 100644 --- a/providers/exasol/docs/index.rst +++ b/providers/exasol/docs/index.rst @@ -42,7 +42,7 @@ :caption: References Python API <_api/airflow/providers/exasol/index> - Tests API <_api/tests/system/exasol/index> + Tests API <_api/tests/system/exasol/index> .. toctree:: :hidden: diff --git a/providers/exasol/tests/system/__init__.py b/providers/exasol/tests/system/__init__.py index e69de29bb2d1d..13a83393a9124 100644 --- a/providers/exasol/tests/system/__init__.py +++ b/providers/exasol/tests/system/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/exasol/tests/system/exasol/__init__.py b/providers/exasol/tests/system/exasol/__init__.py index e69de29bb2d1d..13a83393a9124 100644 --- a/providers/exasol/tests/system/exasol/__init__.py +++ b/providers/exasol/tests/system/exasol/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/exasol/tests/system/exasol/example_exasol.py b/providers/exasol/tests/system/exasol/example_exasol.py index d615733253a16..f44fa0346a9b8 100644 --- a/providers/exasol/tests/system/exasol/example_exasol.py +++ b/providers/exasol/tests/system/exasol/example_exasol.py @@ -1,4 +1,6 @@ # Licensed to the Apache Software Foundation (ASF) under one +# TODO: This license is not consistent with the license used in the project. +# Delete the inconsistent license and above line and rerun pre-commit to insert a good license. # or more contributor license agreements. See the NOTICE file # for additional information # regarding copyright ownership. The ASF licenses this file @@ -89,3 +91,9 @@ # Needed to run the example DAG with pytest test_run = get_test_run(dag) + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From f854795a44e2d82af1b4e4d8f021004db322a8eb Mon Sep 17 00:00:00 2001 From: nailo2c Date: Thu, 24 Apr 2025 10:28:46 -0700 Subject: [PATCH 4/6] fix code duplicate issue --- providers/exasol/docs/index.rst | 1 - providers/exasol/tests/system/exasol/example_exasol.py | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/providers/exasol/docs/index.rst b/providers/exasol/docs/index.rst index 7b21c901c2fcf..0afa727b48387 100644 --- a/providers/exasol/docs/index.rst +++ b/providers/exasol/docs/index.rst @@ -42,7 +42,6 @@ :caption: References Python API <_api/airflow/providers/exasol/index> - Tests API <_api/tests/system/exasol/index> .. toctree:: :hidden: diff --git a/providers/exasol/tests/system/exasol/example_exasol.py b/providers/exasol/tests/system/exasol/example_exasol.py index f44fa0346a9b8..f256d1dbbfb8a 100644 --- a/providers/exasol/tests/system/exasol/example_exasol.py +++ b/providers/exasol/tests/system/exasol/example_exasol.py @@ -1,8 +1,6 @@ # Licensed to the Apache Software Foundation (ASF) under one -# TODO: This license is not consistent with the license used in the project. -# Delete the inconsistent license and above line and rerun pre-commit to insert a good license. # or more contributor license agreements. See the NOTICE file -# for additional information +# distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance @@ -91,9 +89,3 @@ # Needed to run the example DAG with pytest test_run = get_test_run(dag) - - -from tests_common.test_utils.system_tests import get_test_run # noqa: E402 - -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) -test_run = get_test_run(dag) From 37c8c59f27e9340992efbacc26baad5642d269bf Mon Sep 17 00:00:00 2001 From: nailo2c Date: Thu, 24 Apr 2025 10:52:01 -0700 Subject: [PATCH 5/6] fix lint issue #3 --- providers/exasol/docs/index.rst | 7 +++++++ providers/exasol/tests/system/exasol/example_exasol.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/providers/exasol/docs/index.rst b/providers/exasol/docs/index.rst index 0afa727b48387..4841d22dec6a4 100644 --- a/providers/exasol/docs/index.rst +++ b/providers/exasol/docs/index.rst @@ -43,6 +43,13 @@ Python API <_api/airflow/providers/exasol/index> +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System Tests + + System Tests <_api/tests/system/exasol/index> + .. toctree:: :hidden: :maxdepth: 1 diff --git a/providers/exasol/tests/system/exasol/example_exasol.py b/providers/exasol/tests/system/exasol/example_exasol.py index f256d1dbbfb8a..c477aec01f466 100644 --- a/providers/exasol/tests/system/exasol/example_exasol.py +++ b/providers/exasol/tests/system/exasol/example_exasol.py @@ -87,5 +87,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) From 00fb96ce11dd051b2ee84ec92fc545620782cef2 Mon Sep 17 00:00:00 2001 From: nailo2c Date: Thu, 24 Apr 2025 11:15:06 -0700 Subject: [PATCH 6/6] fix CI Static Checks --- providers/exasol/docs/index.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/providers/exasol/docs/index.rst b/providers/exasol/docs/index.rst index 4841d22dec6a4..cf33cd9c50e35 100644 --- a/providers/exasol/docs/index.rst +++ b/providers/exasol/docs/index.rst @@ -46,17 +46,17 @@ .. toctree:: :hidden: :maxdepth: 1 - :caption: System Tests + :caption: Resources - System Tests <_api/tests/system/exasol/index> + PyPI Repository + Installing from sources .. toctree:: :hidden: :maxdepth: 1 - :caption: Resources + :caption: System tests - PyPI Repository - Installing from sources + System Tests <_api/tests/system/exasol/index> .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!