diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index a2461de6285e7..ffd6473031b24 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -41,6 +41,7 @@ body: - apache-pig - apache-pinot - apache-spark + - apache-tinkerpop - apprise - arangodb - asana diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index cff760ad9dc38..cf71336c9486c 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -72,6 +72,9 @@ labelPRBasedOnFilePath: provider:apache-spark: - providers/apache/spark/** + provider:apache-tinkerpop: + - providers/apache/tinkerpop/** + provider:apprise: - providers/apprise/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 964ebe4aa099c..976690a898cba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -246,6 +246,9 @@ repos: name: Check that executables have shebang - id: check-xml name: Check XML files with xmllint + exclude: > + (?x) + ^scripts/ci/docker-compose/gremlin/. - id: trailing-whitespace name: Remove trailing whitespace at end of line exclude: ^airflow-core/docs/img/.*\.dot$|^dev/breeze/doc/images/output.*$ @@ -970,6 +973,7 @@ repos: exclude: > (?x) ^scripts/ci/docker-compose/grafana/.| + ^scripts/ci/docker-compose/gremlin/.| ^scripts/ci/docker-compose/.+-config\.ya?ml$ require_serial: true additional_dependencies: ['jsonschema>=3.2.0,<5.0', 'pyyaml>=6.0.2', 'requests==2.32.3', 'rich>=12.4.4'] diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index 9c360209ac22c..31d15dd324977 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -198,6 +198,8 @@ custom bash/python providers). +---------------------+-----------------------------------------------------+------------------------------------------------+ | apache-spark | ``pip install 'apache-airflow[apache-spark]'`` | All Spark related operators & hooks | +---------------------+-----------------------------------------------------+------------------------------------------------+ +| apache-tinkerpop | ``pip install apache-airflow[apache-tinkerpop]`` | Apache-tinkerpop hooks and operators | ++---------------------+-----------------------------------------------------+------------------------------------------------+ | apache-webhdfs | ``pip install 'apache-airflow[apache-webhdfs]'`` | HDFS hooks and operators | +---------------------+-----------------------------------------------------+------------------------------------------------+ diff --git a/airflow-core/src/airflow/utils/db.py b/airflow-core/src/airflow/utils/db.py index e4919d82bfd36..c7853adfabfbf 100644 --- a/airflow-core/src/airflow/utils/db.py +++ b/airflow-core/src/airflow/utils/db.py @@ -352,6 +352,15 @@ def create_default_connections(session: Session = NEW_SESSION): ), session, ) + merge_conn( + Connection( + conn_id="gremlin_default", + conn_type="gremlin", + host="gremlin", + port=8182, + ), + session, + ) merge_conn( Connection( conn_id="hive_cli_default", diff --git a/contributing-docs/testing/integration_tests.rst b/contributing-docs/testing/integration_tests.rst index 3be9f60a17a65..bc7e9c70f1d32 100644 --- a/contributing-docs/testing/integration_tests.rst +++ b/contributing-docs/testing/integration_tests.rst @@ -64,6 +64,8 @@ core or provider type of test. +--------------+-------------------------------------------------------+ | drill | Integration required for drill operator and hook. | +--------------+-------------------------------------------------------+ +| gremlin | Integration required for gremlin operator and hook. | ++--------------+-------------------------------------------------------+ | kafka | Integration required for Kafka hooks. | +--------------+-------------------------------------------------------+ | kerberos | Integration that provides Kerberos authentication. | diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 2d9668f5a21af..b40f3cbf9b6a4 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -323,7 +323,7 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│kafka | kerberos | keycloak | mongo | mssql |          │ +│gremlin | kafka | kerberos | keycloak | mongo | mssql |│ │openlineage | otel | pinot | qdrant | redis | statsd | │ │trino | ydb)                                           │ │--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index f433121431045..f4421e33dc256 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -194,15 +194,15 @@ breeze build-docs                                                                                                      [OPTIONS] [airbyte | alibaba | all-providers | amazon | apache-airflow | apache-airflow-providers | apache.beam |      apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg |           -apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apprise |       -arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.compat | common.io |         -common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 -| elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | imap -| influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark |                 +apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere |        +common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord |    +docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc |         +hashicorp | helm-chart | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp +| microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |  +pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      +sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata |   +trino | vertica | weaviate | yandex | ydb | zendesk]...                                                                Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 0af15214ce8cc..b8ce7331d5e1e 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -23e0abc8bc26f17cd7e033a3b5654cb4 +8ae2b68e8981f5fc57c32b7e064211bd diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 54d7a14e6adba..b9bf811ef24eb 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -142,15 +142,15 @@ breeze release-management add-back-references                                                                          [OPTIONS] [airbyte | alibaba | all-providers | amazon | apache-airflow | apache-airflow-providers | apache.beam |      apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg |           -apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apprise |       -arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.compat | common.io |         -common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 -| elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | imap -| influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark |                 +apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere |        +common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord |    +docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc |         +hashicorp | helm-chart | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp +| microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |  +pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      +sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata |   +trino | vertica | weaviate | yandex | ydb | zendesk]...                                                                Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index b0761c9ff313b..6373b2f3555d8 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -fbcb4084b1e42f4fb8be2d5a43d81019 +121c5c2424354100827502bc6a6e09c3 diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index e8f786ea94099..17845a419b447 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -143,14 +143,14 @@ breeze release-management generate-issue-content-providers                                                             [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding |     -discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp |  -http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo -| mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill |    -pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | cloudant |     +cncf.kubernetes | cohere | common.compat | common.io | common.messaging | common.sql | databricks | datadog |          +dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |       +google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql |              +microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |       +opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce |    +samba | segment | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram |       +teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                                                     Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index deb7377e876e9..482879d9ca349 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -8539b9fa5e6f545a226180fb2b0479bb +585e7edc334db3e051441aa73afdec7b diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 9f97ecd9e2ef7..075f044461a21 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -197,14 +197,14 @@ breeze release-management prepare-provider-distributions                                                               [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding |     -discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp |  -http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo -| mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill |    -pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | cloudant |     +cncf.kubernetes | cohere | common.compat | common.io | common.messaging | common.sql | databricks | datadog |          +dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |       +google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql |              +microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |       +opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce |    +samba | segment | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram |       +teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                                                     Prepare sdist/whl distributions of Airflow Providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index f5292cbc8cacf..6b9757d044820 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -b4d234fbf771ca1e4b14c92ab7d3e910 +09dd7c6e5817710841fd90c4ac05b4d2 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index 3f64f75e93ce5..20ffd7ee61ce5 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -188,14 +188,14 @@ breeze release-management prepare-provider-documentation                                                               [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |  apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes |      -cohere | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding |     -discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp |  -http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo -| mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill |    -pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | cloudant |     +cncf.kubernetes | cohere | common.compat | common.io | common.messaging | common.sql | databricks | datadog |          +dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |       +google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql |              +microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |       +opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce |    +samba | segment | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram |       +teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                                                     Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index f28c1c909ed51..b29249194c7a4 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -7da34b87ae2b46a281d523e28f4bbfb5 +53c659a34bafdcf7c012eb578dab893f diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index bd70f79c4c176..831be65d4081f 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -199,15 +199,15 @@ breeze release-management publish-docs                                                                                 [OPTIONS] [airbyte | alibaba | all-providers | amazon | apache-airflow | apache-airflow-providers | apache.beam |      apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg |           -apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apprise |       -arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.compat | common.io |         -common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 -| elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | imap -| influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark |                 +apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere |        +common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord |    +docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc |         +hashicorp | helm-chart | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp +| microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |  +pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      +sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata |   +trino | vertica | weaviate | yandex | ydb | zendesk]...                                                                Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index fc7cd0b3bbc65..c0f2a006df77e 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -f50847005b110bf0474ba730172efd8b +4051081e930cd2c899a8db9c614d0ed2 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index c591c16de3aa1..d37cb29f1ccf1 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -188,12 +188,12 @@ │--provider-idProvider id to generate the requirements for                                                  │ │(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid |  │ │apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka |    │ -│apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apprise | arangodb |  │ -│asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.compat |       │ -│common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding |     │ -│discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |     │ -│google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure |       │ -│microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |  │ +│apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop |    │ +│apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere |  │ +│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |│ +│dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git |   │ +│github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure│ +│| microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |│ │openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |  │ │pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp│ │| singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram |     │ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index 1c70f48771cde..7799d72651b51 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -9df7dccf519dc82ef16b95a8102936fd +781a886d7ccd2537dd9af7a479bb3007 diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index ff42b6fcd75ab..c33e37ad242be 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -547,7 +547,7 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│kafka | kerberos | keycloak | mongo | mssql |          │ +│gremlin | kafka | kerberos | keycloak | mongo | mssql |│ │openlineage | otel | pinot | qdrant | redis | statsd | │ │trino | ydb)                                           │ │--load-example-dags-eEnable configuration to load example DAGs when starting│ diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index bc1afa16956f4..4bc4dbed2a13d 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -19ef098d3ebe4f9a7d5a4c805600bcfd +664f71ecf40d322c88efa458a7e1a564 diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index 9cb6a3acdfbf1..68db41092407c 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -441,7 +441,7 @@ │--integrationCore Integrations to enable when running (can be more  │ │than one).                                             │ │(all | all-testable | cassandra | celery | drill |     │ -│kafka | kerberos | keycloak | mongo | mssql |          │ +│gremlin | kafka | kerberos | keycloak | mongo | mssql |│ │openlineage | otel | pinot | qdrant | redis | statsd | │ │trino | ydb)                                           │ │--standalone-dag-processor/--no-standalone-dag-processoRun standalone dag processor for start-airflow         │ diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 34e5ac548698a..12d99f7aa940b 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -d17743200dad3972e3432e9c2e91ba02 +bae200412140579f690c72361f82f838 diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg index 4f85c968d2203..71bce41db52bd 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.svg +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.svg @@ -237,8 +237,8 @@ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Integration tests ──────────────────────────────────────────────────────────────────────────────────────────────────╮ │--integrationProviders Integration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | drill | kafka | mongo | mssql | openlineage | pinot |   │ -│qdrant | redis | trino | ydb)                                                                      │ +│(all | all-testable | cassandra | celery | drill | gremlin | kafka | mongo | mssql | openlineage | │ +│pinot | qdrant | redis | trino | ydb)                                                              │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ diff --git a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt index 4796ca1d55b1d..1b6995e16c3d1 100644 --- a/dev/breeze/doc/images/output_testing_providers-integration-tests.txt +++ b/dev/breeze/doc/images/output_testing_providers-integration-tests.txt @@ -1 +1 @@ -a384620e2acf2a0f693c742a7b1d41a5 +48aafaf7c9282ae70307e168ffefb0ab diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index d131afb86482a..5910f86ec811f 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -64,6 +64,7 @@ "celery", "cassandra", "drill", + "gremlin", "kafka", "mongo", "mssql", @@ -353,6 +354,7 @@ def get_default_platform_machine() -> str: DRILL_HOST_PORT = "28047" FLOWER_HOST_PORT = "25555" +GREMLIN_HOST_PORT = "8182" MSSQL_HOST_PORT = "21433" MYSQL_HOST_PORT = "23306" POSTGRES_HOST_PORT = "25433" diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 363078439b5b2..947ac2c54f222 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -46,6 +46,7 @@ EDGE_EXECUTOR, FAB_AUTH_MANAGER, FLOWER_HOST_PORT, + GREMLIN_HOST_PORT, KEYCLOAK_INTEGRATION, MOUNT_ALL, MOUNT_PROVIDERS_AND_TESTS, @@ -581,6 +582,7 @@ def env_variables_for_docker_commands(self) -> dict[str, str]: _set_var(_env, "DRILL_HOST_PORT", None, DRILL_HOST_PORT) _set_var(_env, "ENABLE_COVERAGE", self.enable_coverage) _set_var(_env, "FLOWER_HOST_PORT", None, FLOWER_HOST_PORT) + _set_var(_env, "GREMLIN_HOST_PORT", None, GREMLIN_HOST_PORT) _set_var(_env, "EXCLUDED_PROVIDERS", self.excluded_providers) _set_var(_env, "FORCE_LOWEST_DEPENDENCIES", self.force_lowest_dependencies) _set_var(_env, "SQLALCHEMY_WARN_20", self.force_sa_warnings) diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index 7a1ef48f21535..fa0e48f519790 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -63,6 +63,7 @@ def _find_all_integration_folders() -> list[str]: "providers/apache/hive/tests/integration", "providers/apache/kafka/tests/integration", "providers/apache/pinot/tests/integration", + "providers/apache/tinkerpop/tests/integration", "providers/celery/tests/integration", "providers/google/tests/integration", "providers/microsoft/mssql/tests/integration", diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 3469ee95168a9..6a87e8146b3a0 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1123,8 +1123,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "core-test-types-list-as-strings-in-json": ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON, "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "testable-core-integrations": "['kerberos']", - "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'kafka', 'mongo', " - "'pinot', 'qdrant', 'redis', 'trino', 'ydb']", + "testable-providers-integrations": "['celery', 'cassandra', 'drill', 'gremlin', 'kafka', " + "'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']", "needs-mypy": "true", "mypy-checks": ALL_MYPY_CHECKS, }, diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 93514805fd598..b5275a37c8c69 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1813,6 +1813,9 @@ timedeltas timeframe timespan timezones +TinkerPop +Tinkerpop +tinkerpop tis TLS tls diff --git a/providers/apache/tinkerpop/README.rst b/providers/apache/tinkerpop/README.rst new file mode 100644 index 0000000000000..5ac5a4238690e --- /dev/null +++ b/providers/apache/tinkerpop/README.rst @@ -0,0 +1,63 @@ + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY + +Package ``apache-airflow-providers-apache-tinkerpop`` + +Release: ``1.0.0`` + + +`Apache TinkerPop `__. +Apache TinkerPop is a graph computing framework for both graph databases (OLTP) and graph analytic +systems (OLAP) and Gremlin is its graph traversal language. + + +Provider package +---------------- + +This is a provider package for ``apache.tinkerpop`` provider. All classes for this provider package +are in ``airflow.providers.apache.tinkerpop`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-apache-tinkerpop`` + +The package supports the following python versions: 3.9,3.10,3.11,3.12 + +Requirements +------------ + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=2.9.0`` +``gremlinpython`` ``>=3.7.0`` +================== ================== + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/apache/tinkerpop/docs/changelog.rst b/providers/apache/tinkerpop/docs/changelog.rst new file mode 100644 index 0000000000000..dbb7f2a36d5ae --- /dev/null +++ b/providers/apache/tinkerpop/docs/changelog.rst @@ -0,0 +1,24 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-apache-tinkerpop`` + + +1.0.0 +..... + +* ``Initial version of the provider. (#47466)`` diff --git a/providers/apache/tinkerpop/docs/commits.rst b/providers/apache/tinkerpop/docs/commits.rst new file mode 100644 index 0000000000000..b7a280ec6bdc6 --- /dev/null +++ b/providers/apache/tinkerpop/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-apache-tinkerpop +------------------------------------------------------ + +`Apache TinkerPop `__. +Apache TinkerPop is a graph computing framework for both graph databases (OLTP) and graph analytic +systems (OLAP) and Gremlin is its graph traversal language. + + +This is detailed commit list of changes for versions provider package: ``apache.tinkerpop``. +For high-level changelog, see :doc:`package information including changelog `. diff --git a/providers/apache/tinkerpop/docs/conf.py b/providers/apache/tinkerpop/docs/conf.py new file mode 100644 index 0000000000000..377a7626ba9d1 --- /dev/null +++ b/providers/apache/tinkerpop/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-apache-tinkerpop" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/apache/tinkerpop/docs/connections/tinkerpop.rst b/providers/apache/tinkerpop/docs/connections/tinkerpop.rst new file mode 100644 index 0000000000000..ce64ad135cd8c --- /dev/null +++ b/providers/apache/tinkerpop/docs/connections/tinkerpop.rst @@ -0,0 +1,88 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/connection:tinkerpop: + +Gremlin Connection +==================== + +The Gremlin connection type enables integrations with Gremlin Server. + +Authenticating to Gremlin +--------------------------- + +Authenticate to Gremlin using the `Germlin python client default authentication +`_. + +Default Connection IDs +---------------------- + +Hooks and operators related to Gremlin use ``gremlin_default`` by default. + +Configuring the Connection +-------------------------- + +Use the ``gremlin_conn_id`` argument to connect to your Gremlin instance where +the connection metadata is structured as follows: + +.. list-table:: Gremlin Airflow Connection Metadata + :widths: 25 25 + :header-rows: 1 + + * - Parameter + - Input + * - Host: string + - Gremlin hostname + * - Port: int + - Gremlin port + * - Username: string + - Gremlin user + * - Password: string + - Gremlin user password + * - Scheme: string + - Gremlin Scheme + * - Schema: string + - Gremlin Schema + +URI format example +^^^^^^^^^^^^^^^^^^ + +If serializing with Airflow URI: + +.. code-block:: bash + + export AIRFLOW_CONN_GREMLIN_DEFAULT='gremlin://username:password@host:port/schema' + +Note that all components of the URI should be URL-encoded. + +JSON format example +^^^^^^^^^^^^^^^^^^^ + +If serializing with JSON: + +.. code-block:: bash + + export AIRFLOW_CONN_GREMLIN_DEFAULT='{ + "conn_type": "gremlin", + "login": "username", + "host": "host", + "password": "password", + "schema": "schema", + "port": 433 + }' diff --git a/providers/apache/tinkerpop/docs/index.rst b/providers/apache/tinkerpop/docs/index.rst new file mode 100644 index 0000000000000..1205b73736495 --- /dev/null +++ b/providers/apache/tinkerpop/docs/index.rst @@ -0,0 +1,107 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Package ``apache-airflow-providers-apache-tinkerpop`` +===================================================== + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection Types + Operators + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Python API <_api/airflow/providers/apache/tinkerpop/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/apache/tinkerpop/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + Example DAGs + PyPI Repository + Installing from sources + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-apache-tinkerpop package +------------------------------------------------------ + +`Apache TinkerPop `__. +Apache TinkerPop is a graph computing framework for both graph databases (OLTP) and graph analytic +systems (OLAP) and Gremlin is its graph traversal language. + + +Release: 1.0.0 + +Provider package +---------------- + +This package is for the ``apache.tinkerpop`` provider. +All classes for this package are included in the ``airflow.providers.apache.tinkerpop`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation via +``pip install apache-airflow-providers-apache-tinkerpop``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider package is ``2.9.0``. + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=2.9.0`` +``gremlinpython`` ``>=3.7.0`` +``nest-asyncio`` ``>=1.6.0`` +================== ================== diff --git a/providers/apache/tinkerpop/docs/installing-providers-from-sources.rst b/providers/apache/tinkerpop/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..fdbb17d017579 --- /dev/null +++ b/providers/apache/tinkerpop/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/apache/tinkerpop/docs/operators/tinkerpop.rst b/providers/apache/tinkerpop/docs/operators/tinkerpop.rst new file mode 100644 index 0000000000000..cb1e73addb480 --- /dev/null +++ b/providers/apache/tinkerpop/docs/operators/tinkerpop.rst @@ -0,0 +1,46 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Apache TinkerPop Operators +========================== + +.. _howto/operator:`GremlinOperator`: + +GremlinOperator +--------------- + +Executes Gremlin queries on a remote Gremlin Server. + +For parameter definition take a look at :class:`~airflow.providers.apache.tinkerpop.operators.gremlin.GremlinOperator`. + +Using the operator +"""""""""""""""""" + +An example usage of the GremlinOperator to query from gremlin server is as follows: + +.. exampleinclude:: /../tests/system/apache/tinkerpop/example_gremlin_dag.py + :language: python + :dedent: 4 + :start-after: [START run_query_gremlin_operator] + :end-before: [END run_query_gremlin_operator] + + +Reference +""""""""" + +For further information, look at: https://tinkerpop.apache.org/gremlin.html diff --git a/providers/apache/tinkerpop/docs/security.rst b/providers/apache/tinkerpop/docs/security.rst new file mode 100644 index 0000000000000..351ff007ebf2f --- /dev/null +++ b/providers/apache/tinkerpop/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/apache/tinkerpop/provider.yaml b/providers/apache/tinkerpop/provider.yaml new file mode 100644 index 0000000000000..f61e5993e0275 --- /dev/null +++ b/providers/apache/tinkerpop/provider.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +package-name: apache-airflow-providers-apache-tinkerpop +name: Apache TinkerPop +description: | + `Apache TinkerPop `__. + Apache TinkerPop is a graph computing framework for both graph databases (OLTP) and graph analytic + systems (OLAP) and Gremlin is its graph traversal language. +state: ready +source-date-epoch: 1686592800 +# note that these versions are maintained by the release manager - do not update them manually +versions: + - 1.0.0 +integrations: + - integration-name: Apache TinkerPop + external-doc-url: https://tinkerpop.apache.org/ + how-to-guide: + - /docs/apache-airflow-providers-apache-tinkerpop/operators/tinkerpop.rst + tags: [apache] +operators: + - integration-name: Apache TinkerPop + python-modules: + - airflow.providers.apache.tinkerpop.operators.gremlin +hooks: + - integration-name: Apache TinkerPop + python-modules: + - airflow.providers.apache.tinkerpop.hooks.gremlin +connection-types: + - hook-class-name: airflow.providers.apache.tinkerpop.hooks.gremlin.GremlinHook + connection-type: gremlin diff --git a/providers/apache/tinkerpop/pyproject.toml b/providers/apache/tinkerpop/pyproject.toml new file mode 100644 index 0000000000000..58a86b8d21fcd --- /dev/null +++ b/providers/apache/tinkerpop/pyproject.toml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-apache-tinkerpop" +version = "1.0.0" +description = "Provider package apache-airflow-providers-apache-tinkerpop for Apache Airflow" +readme = "README.rst" +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "apache.tinkerpop", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] +requires-python = "~=3.9" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=2.9.0", + "gremlinpython>=3.7.0", +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + # Additional devel dependencies (do not remove this line and add extra development dependencies) +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-tinkerpop/1.0.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-tinkerpop/1.0.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.apache.tinkerpop.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.apache.tinkerpop" diff --git a/providers/apache/tinkerpop/src/airflow/__init__.py b/providers/apache/tinkerpop/src/airflow/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/tinkerpop/src/airflow/providers/__init__.py b/providers/apache/tinkerpop/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/__init__.py b/providers/apache/tinkerpop/src/airflow/providers/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/LICENSE b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/__init__.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/__init__.py new file mode 100644 index 0000000000000..58f646b31f81e --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "1.0.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "2.9.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-apache-tinkerpop:{__version__}` needs Apache Airflow 2.9.0+" + ) diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/get_provider_info.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/get_provider_info.py new file mode 100644 index 0000000000000..136b01fc499cc --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/get_provider_info.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-apache-tinkerpop", + "name": "Apache TinkerPop", + "description": "`Apache TinkerPop `__.\nApache TinkerPop is a graph computing framework for both graph databases (OLTP) and graph analytic\nsystems (OLAP) and Gremlin is its graph traversal language.\n", + "integrations": [ + { + "integration-name": "Apache TinkerPop", + "external-doc-url": "https://tinkerpop.apache.org/", + "how-to-guide": ["/docs/apache-airflow-providers-apache-tinkerpop/operators/tinkerpop.rst"], + "tags": ["apache"], + } + ], + "operators": [ + { + "integration-name": "Apache TinkerPop", + "python-modules": ["airflow.providers.apache.tinkerpop.operators.gremlin"], + } + ], + "hooks": [ + { + "integration-name": "Apache TinkerPop", + "python-modules": ["airflow.providers.apache.tinkerpop.hooks.gremlin"], + } + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.apache.tinkerpop.hooks.gremlin.GremlinHook", + "connection-type": "gremlin", + } + ], + } diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/hooks/__init__.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/hooks/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/hooks/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/hooks/gremlin.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/hooks/gremlin.py new file mode 100644 index 0000000000000..3c547a02c338c --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/hooks/gremlin.py @@ -0,0 +1,154 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This module allows connecting to an Graph DB using the Gremlin Client.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from gremlin_python.driver.client import Client + +from airflow.hooks.base import BaseHook + +if TYPE_CHECKING: + from airflow.models import Connection + + +logger = logging.getLogger(__name__) + + +class GremlinHook(BaseHook): + """ + Interact with Graph DB using the Gremlin Client. + + This hook creates a connection to Graph DB and allows you to run Gremlin queries.` + + :param gremlin_conn_id: Reference to the connection ID configured in Airflow. + """ + + conn_name_attr = "gremlin__conn_id" + default_conn_name = "gremlin_default" + conn_type = "gremlin" + hook_name = "Gremlin" + default_port = 443 + traversal_source = "g" + + def __init__(self, conn_id: str = default_conn_name, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.gremlin_conn_id = conn_id + self.connection = kwargs.pop("connection", None) + self.client: Client | None = None + + def get_conn(self, serializer=None) -> Client: + """ + Establish a connection to Graph DB with the Gremlin Client. + + :param serializer: Message serializer to use for the client. + + :return: An instance of the Gremlin Client. + """ + if self.client is not None: + return self.client + + self.connection = self.get_connection(self.gremlin_conn_id) + + uri = self.get_uri(self.connection) + self.log.info("Connecting to URI: %s", uri) + + self.client = self.get_client( + self.connection, self.traversal_source, uri, message_serializer=serializer + ) + return self.client + + def get_uri(self, conn: Connection) -> str: + """ + Build the URI from the connection object and extra parameters. + + :param conn: Airflow Connection object. + + :return: URI string. + """ + # For Graph DB using Gremlin, the secure WebSocket scheme is typically "wss" + scheme = "wss" if conn.conn_type == "gremlin" else "ws" + host = conn.host + port = conn.port if conn.port is not None else self.default_port + schema = "" if conn.conn_type == "gremlin" else "gremlin" + return f"{scheme}://{host}:{port}/{schema}" + + def get_client( + self, conn: Connection, traversal_source: str, uri: str, message_serializer=None + ) -> Client: + """ + Create and return a new Gremlin client. + + :param conn: Airflow Connection object. + :param traversal_source: Traversal source for the Gremlin client. + :param uri: URI string for connecting to Graph DB. + :param message_serializer: Message serializer to use for the client. + + :return: An instance of the Gremlin Client. + """ + # Build the username. This example uses the connection's schema and login. + login = conn.login if conn.login not in ["mylogin", None] else "" + schema = conn.schema if conn.schema not in ["gremlin", None] else "" + password = conn.password if conn.password not in ["mysecret", None] else "" + username = f"/dbs/{login}/colls/{schema}" if login and schema else "" + # Build the kwargs for the Client. + client_kwargs = { + "url": uri, + "traversal_source": traversal_source, + "username": username, + "password": password, + } + + # If a serializer is provided, check if it's a type and instantiate it. + if message_serializer is not None: + if isinstance(message_serializer, type): + message_serializer = message_serializer() + client_kwargs["message_serializer"] = message_serializer + + return Client(**client_kwargs) + + def run(self, query: str, serializer=None, bindings=None, request_options=None) -> list[Any]: + """ + Execute a Gremlin query and return the results. + + :param query: Gremlin query string. + :param serializer: Message serializer to use for the query. + :param bindings: Bindings to use for the query. + :param request_options: Request options to use for the query. + + :return: List containing the query results. + """ + client = self.get_conn(serializer) + + try: + results_list = ( + client.submit(message=query, bindings=bindings, request_options=request_options) + .all() + .result() + ) + except Exception as e: + logger.error("An error occurred while running the query: %s", str(e)) + raise e + finally: + if client is not None: + client.close() + self.client = None + + return results_list diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/operators/__init__.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/operators/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/operators/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/operators/gremlin.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/operators/gremlin.py new file mode 100644 index 0000000000000..f7064890d5d4b --- /dev/null +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/operators/gremlin.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from airflow.models import BaseOperator +from airflow.providers.apache.tinkerpop.hooks.gremlin import GremlinHook + +if TYPE_CHECKING: + try: + from airflow.sdk.definitions.context import Context + except ImportError: + # TODO: Remove once provider drops support for Airflow 2 + from airflow.utils.context import Context + + +class GremlinOperator(BaseOperator): + """ + Execute a Gremlin query. + + :param query: The Gremlin query to execute. + :param gremlin_conn_id: The connection ID to use when connecting to Gremlin. Defaults to "gremlin_default". + """ + + template_fields = ("query",) + + def __init__(self, query: str, gremlin_conn_id: str = "gremlin_default", **kwargs) -> None: + super().__init__(**kwargs) + self.query = query + self.gremlin_conn_id = gremlin_conn_id + + def execute(self, context: Context) -> Any: + hook = GremlinHook(conn_id=self.gremlin_conn_id) + # Note: the hook method is defined as run() in our hook implementation. + # If you prefer, you can add an alias run_query = run in your hook. + try: + return hook.run(self.query) + finally: + if hasattr(hook, "client") and hook.client: + hook.client.close() # Ensure client cleanup diff --git a/providers/apache/tinkerpop/tests/conftest.py b/providers/apache/tinkerpop/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/apache/tinkerpop/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/apache/tinkerpop/tests/integration/__init__.py b/providers/apache/tinkerpop/tests/integration/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/tinkerpop/tests/integration/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/integration/apache/__init__.py b/providers/apache/tinkerpop/tests/integration/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/tinkerpop/tests/integration/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/__init__.py b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/__init__.py b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/test_gremlin.py b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/test_gremlin.py new file mode 100644 index 0000000000000..2a2ea25f95625 --- /dev/null +++ b/providers/apache/tinkerpop/tests/integration/apache/tinkerpop/hooks/test_gremlin.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os + +import pytest +from gremlin_python.driver.serializer import GraphSONSerializersV2d0 + +from airflow.providers.apache.tinkerpop.hooks.gremlin import GremlinHook + +AIRFLOW_CONN_GREMLIN_DEFAULT = "ws://mylogin:mysecret@gremlin:8182/gremlin" + + +@pytest.mark.integration("gremlin") +class TestGremlinHook: + def setup_method(self): + os.environ["AIRFLOW_CONN_GREMLIN_DEFAULT"] = AIRFLOW_CONN_GREMLIN_DEFAULT + self.hook = GremlinHook() + add_query = "g.addV('person').property('id', 'person1').property('name', 'Alice')" + self.hook.run(add_query) + + def teardown_method(self): + self.hook.run("g.V().drop().iterate()") + + def test_another_query(self): + result = self.hook.run("g.V().hasLabel('person').count()") + assert isinstance(result, list) + + def test_run(self): + result = self.hook.run( + "g.V().hasLabel('person').valueMap(true)", serializer=GraphSONSerializersV2d0() + ) + expected = "[{'id': ['person1'], 'label': 'person', 'name': ['Alice']}]" + assert str(result) == expected diff --git a/providers/apache/tinkerpop/tests/system/__init__.py b/providers/apache/tinkerpop/tests/system/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/tinkerpop/tests/system/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/system/apache/__init__.py b/providers/apache/tinkerpop/tests/system/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/tinkerpop/tests/system/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/tinkerpop/tests/system/apache/tinkerpop/__init__.py b/providers/apache/tinkerpop/tests/system/apache/tinkerpop/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/tinkerpop/tests/system/apache/tinkerpop/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/system/apache/tinkerpop/example_gremlin_dag.py b/providers/apache/tinkerpop/tests/system/apache/tinkerpop/example_gremlin_dag.py new file mode 100644 index 0000000000000..94f3e304125c6 --- /dev/null +++ b/providers/apache/tinkerpop/tests/system/apache/tinkerpop/example_gremlin_dag.py @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example use of Gremlin related operators. +""" + +from __future__ import annotations + +import os +from datetime import datetime + +from airflow import DAG +from airflow.providers.apache.tinkerpop.operators.gremlin import GremlinOperator + +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_gremlin_dag" + + +with DAG( + DAG_ID, + start_date=datetime(2021, 1, 1), + schedule=None, + tags=["example"], + catchup=False, +) as dag: + # [START run_query_gremlin_operator] + + gremlin_task = GremlinOperator( + task_id="run_gremlin_query", gremlin_conn_id="gremlin_conn_id", query="g.V()" + ) + + # [END run_query_gremlin_operator] + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/providers/apache/tinkerpop/tests/unit/__init__.py b/providers/apache/tinkerpop/tests/unit/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/tinkerpop/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/tinkerpop/tests/unit/apache/__init__.py b/providers/apache/tinkerpop/tests/unit/apache/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/apache/tinkerpop/tests/unit/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/__init__.py b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/hooks/__init__.py b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/hooks/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/hooks/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/hooks/test_gremlin.py b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/hooks/test_gremlin.py new file mode 100644 index 0000000000000..d002dcb2f609b --- /dev/null +++ b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/hooks/test_gremlin.py @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.models import Connection +from airflow.providers.apache.tinkerpop.hooks.gremlin import GremlinHook + + +@pytest.fixture +def gremlin_hook(): + """Fixture to provide a GremlinHook instance with proper teardown.""" + hook = GremlinHook() + yield hook + # Teardown: Ensure the client is closed if it exists + if hook.client: + hook.client.close() + + +class TestGremlinHook: + @pytest.mark.parametrize( + "host, port, expected_uri", + [ + ("host", None, "ws://host:443/gremlin"), + ("myhost", 1234, "ws://myhost:1234/gremlin"), + ("localhost", 8888, "ws://localhost:8888/gremlin"), + ], + ) + def test_get_uri(self, host, port, expected_uri, gremlin_hook): + """ + Test that get_uri builds the expected URI from the connection. + """ + conn = Connection(conn_id="gremlin_default", host=host, port=port) + with mock.patch.dict("os.environ", AIRFLOW_CONN_GREMLIN_DEFAULT=conn.get_uri()): + uri = gremlin_hook.get_uri(conn) + + assert uri == expected_uri + + def test_get_conn(self, gremlin_hook): + """ + Test that get_conn() retrieves the connection and creates a client correctly. + """ + conn = Connection( + conn_type="gremlin", + conn_id="gremlin_default", + host="host", + port=1234, + schema="mydb", + login="login", + password="mypassword", + ) + gremlin_hook.get_connection = lambda conn_id: conn + + with mock.patch("airflow.providers.apache.tinkerpop.hooks.gremlin.Client") as mock_client: + gremlin_hook.get_conn() + expected_uri = "wss://host:1234/" + expected_username = "/dbs/login/colls/mydb" + + mock_client.assert_called_once_with( + url=expected_uri, + traversal_source=gremlin_hook.traversal_source, + username=expected_username, + password="mypassword", + ) + + @pytest.mark.parametrize( + "serializer, should_include", + [ + (None, False), + ("dummy_serializer", True), + ], + ) + def test_get_client_message_serializer(self, serializer, should_include, gremlin_hook): + """ + Test that get_client() includes message_serializer only when provided. + """ + conn = Connection( + conn_id="gremlin_default", + host="host", + port=1234, + schema="mydb", + login="login", + password="mypassword", + ) + uri = "wss://test.uri" + traversal_source = "g" + + with mock.patch("airflow.providers.apache.tinkerpop.hooks.gremlin.Client") as mock_client: + gremlin_hook.get_client(conn, traversal_source, uri, message_serializer=serializer) + call_args = mock_client.call_args.kwargs + if should_include: + assert "message_serializer" in call_args + assert call_args["message_serializer"] == serializer + else: + assert "message_serializer" not in call_args + + @pytest.mark.parametrize( + "side_effect, expected_exception, expected_result", + [ + (None, None, ["dummy_result"]), + (Exception("Test error"), Exception, None), + ], + ) + def test_run(self, side_effect, expected_exception, expected_result, gremlin_hook): + """ + Test that run() returns the expected result or propagates an exception, with proper cleanup. + """ + query = "g.V().limit(1)" + + # Mock the client instance + with mock.patch("airflow.providers.apache.tinkerpop.hooks.gremlin.Client") as mock_client: + instance = mock_client.return_value + if side_effect is None: + instance.submit.return_value.all.return_value.result.return_value = expected_result + else: + instance.submit.return_value.all.return_value.result.side_effect = side_effect + + # Mock get_connection to simplify setup + conn = Connection( + conn_id="gremlin_default", + host="host", + port=1234, + schema="mydb", + login="login", + password="mypassword", + ) + gremlin_hook.get_connection = lambda conn_id: conn + + if expected_exception: + with pytest.raises(expected_exception, match="Test error"): + gremlin_hook.run(query) + else: + result = gremlin_hook.run(query) + assert result == expected_result diff --git a/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/operators/__init__.py b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/operators/__init__.py new file mode 100644 index 0000000000000..217e5db960782 --- /dev/null +++ b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/operators/__init__.py @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/operators/test_gremlin.py b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/operators/test_gremlin.py new file mode 100644 index 0000000000000..59fe2f6c50457 --- /dev/null +++ b/providers/apache/tinkerpop/tests/unit/apache/tinkerpop/operators/test_gremlin.py @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from unittest import mock + +from airflow.models import Connection +from airflow.providers.apache.tinkerpop.operators.gremlin import GremlinOperator +from airflow.utils import timezone + +DEFAULT_DATE = timezone.datetime(2015, 1, 1) +TEST_DAG_ID = "unit_test_dag" + + +class TestGremlinOperator: + @mock.patch("airflow.providers.apache.tinkerpop.operators.gremlin.GremlinHook") + def test_gremlin_operator(self, mock_hook): + """ + Test that the GremlinOperator instantiates the hook with the proper connection id + and calls its run() method with the provided query. + """ + query = "g.V().limit(1)" + op = GremlinOperator(task_id="basic_gremlin", query=query, gremlin_conn_id="gremlin_default") + + # Create a dummy context + context = mock.MagicMock() + + # Create a dummy connection + dummy_conn = Connection( + conn_id="gremlin_default", + host="host", + port=443, + schema="mydb", + login="mylogin", + password="mypassword", + ) + + # Mock hook instance + mock_hook_instance = mock_hook.return_value + mock_hook_instance.get_connection.return_value = dummy_conn + mock_hook_instance.run.return_value = None + mock_hook_instance.client = mock.MagicMock() # Mock client attribute + mock_hook_instance.client.close = mock.MagicMock() # Mock close method + + # Execute the operator + op.execute(context) + + # Verify hook instantiation and run call + mock_hook.assert_called_once_with(conn_id="gremlin_default") + mock_hook_instance.run.assert_called_once_with(query) + mock_hook_instance.client.close.assert_called_once() # Verify cleanup diff --git a/pyproject.toml b/pyproject.toml index 27266a56e006e..07389183aacf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,6 +178,9 @@ packages = [] "apache.spark" = [ "apache-airflow-providers-apache-spark>=4.11.1" ] +"apache.tinkerpop" = [ + "apache-airflow-providers-apache-tinkerpop" +] "apprise" = [ "apache-airflow-providers-apprise>=1.4.1" ] @@ -429,6 +432,7 @@ packages = [] "apache-airflow-providers-apache-pig>=4.6.0", "apache-airflow-providers-apache-pinot>=4.5.1", "apache-airflow-providers-apache-spark>=4.11.1", + "apache-airflow-providers-apache-tinkerpop", "apache-airflow-providers-apprise>=1.4.1", "apache-airflow-providers-arangodb>=2.7.0", "apache-airflow-providers-asana>=2.7.0", @@ -998,6 +1002,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/apache/pinot/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/spark/src", "$MYPY_CONFIG_FILE_DIR/providers/apache/spark/tests", + "$MYPY_CONFIG_FILE_DIR/providers/apache/tinkerpop/src", + "$MYPY_CONFIG_FILE_DIR/providers/apache/tinkerpop/tests", "$MYPY_CONFIG_FILE_DIR/providers/apprise/src", "$MYPY_CONFIG_FILE_DIR/providers/apprise/tests", "$MYPY_CONFIG_FILE_DIR/providers/arangodb/src", @@ -1268,6 +1274,7 @@ apache-airflow-providers-apache-livy = { workspace = true } apache-airflow-providers-apache-pig = { workspace = true } apache-airflow-providers-apache-pinot = { workspace = true } apache-airflow-providers-apache-spark = { workspace = true } +apache-airflow-providers-apache-tinkerpop = { workspace = true } apache-airflow-providers-apprise = { workspace = true } apache-airflow-providers-arangodb = { workspace = true } apache-airflow-providers-asana = { workspace = true } @@ -1379,6 +1386,7 @@ members = [ "providers/apache/pig", "providers/apache/pinot", "providers/apache/spark", + "providers/apache/tinkerpop", "providers/apprise", "providers/arangodb", "providers/asana", diff --git a/scripts/ci/docker-compose/gremlin/graph.properties b/scripts/ci/docker-compose/gremlin/graph.properties new file mode 100644 index 0000000000000..e8550156de7bd --- /dev/null +++ b/scripts/ci/docker-compose/gremlin/graph.properties @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Specify the graph implementation class +gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph + +# Optional: Specify a file location for persistent storage. +# Uncomment the following line if you wish to persist graph data between restarts. +gremlin.tinkergraph.graphLocation=/opt/gremlin-server/data/graph.json + +# Configure the ID managers for vertices and edges +gremlin.tinkergraph.vertexIdManager=LONG +gremlin.tinkergraph.graphFormat=graphson diff --git a/scripts/ci/docker-compose/gremlin/gremlin-entrypoint.sh b/scripts/ci/docker-compose/gremlin/gremlin-entrypoint.sh new file mode 100755 index 0000000000000..2c58e7b518c96 --- /dev/null +++ b/scripts/ci/docker-compose/gremlin/gremlin-entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#!/bin/sh +set -eu + +# Fix permissions on the config directory +echo "Fixing permissions for /opt/gremlin-server/conf..." +chmod -R a+rw /opt/gremlin-server/conf +ls -la /opt/gremlin-server/conf + +# Start Gremlin Server in the background +echo "Starting Gremlin Server on port 8182..." +cd /opt/gremlin-server || exit +./bin/gremlin-server.sh conf/gremlin-server.yaml & + +# Wait for Gremlin to be ready +echo "Waiting for Gremlin Server to start on port 8182..." +while ! nc -z gremlin 8182 2>/dev/null; do + echo "Gremlin still not started" + sleep 5 +done +sleep 3 +echo "Gremlin Server is running" + +# Keep the container running +wait diff --git a/scripts/ci/docker-compose/gremlin/gremlin-server.yaml b/scripts/ci/docker-compose/gremlin/gremlin-server.yaml new file mode 100644 index 0000000000000..92e623dcb7a12 --- /dev/null +++ b/scripts/ci/docker-compose/gremlin/gremlin-server.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +host: 172.18.0.2 +port: 8182 +channelizer: org.apache.tinkerpop.gremlin.server.channel.WebSocketChannelizer +graphs: { + graph: conf/graph.properties} +scriptEngines: { + gremlin-groovy: { + plugins: { + org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.tinkergraph.jsr223.TinkerGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: { + classImports: [java.lang.Math], methodImports: [java.lang.Math#*] + }, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: { + files: [scripts/empty-sample.groovy] + }}}} +serializers: + - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + config: {includeTypes: true}} + - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + config: {ioRegistries: [org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerIoRegistryV3d0]}} + - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1} + - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + config: {serializeResultToString: true}} +processors: + - {className: org.apache.tinkerpop.gremlin.server.op.session.SessionOpProcessor, + config: {sessionTimeout: 28800000}} + - {className: org.apache.tinkerpop.gremlin.server.op.traversal.TraversalOpProcessor, + config: {cacheExpirationTime: 600000, cacheMaxSize: 1000}} +metrics: { + consoleReporter: {enabled: true, interval: 180000}, + csvReporter: {enabled: true, interval: 180000, fileName: /tmp/gremlin-server-metrics.csv}, + jmxReporter: {enabled: true}, + slf4jReporter: {enabled: true, interval: 180000}} +strictTransactionManagement: false +idleConnectionTimeout: 0 +keepAliveInterval: 0 +maxInitialLineLength: 4096 +maxHeaderSize: 8192 +maxChunkSize: 8192 +maxContentLength: 65536 +maxAccumulationBufferComponents: 1024 +resultIterationBatchSize: 64 +writeBufferLowWaterMark: 32768 +writeBufferHighWaterMark: 65536 +ssl: {enabled: false} diff --git a/scripts/ci/docker-compose/gremlin/log4j-server.properties b/scripts/ci/docker-compose/gremlin/log4j-server.properties new file mode 100644 index 0000000000000..e2d23083842ae --- /dev/null +++ b/scripts/ci/docker-compose/gremlin/log4j-server.properties @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log4j.rootLogger=INFO, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%p] %C{1} - %m%n + +log4j.logger.org.apache.tinkerpop.gremlin.driver.Connection=OFF +log4j.logger.org.apache.tinkerpop.gremlin.driver.ConnectionPool=OFF +log4j.logger.org.apache.tinkerpop.gremlin.neo4j.structure.Neo4jGraph=ERROR +log4j.logger.org.apache.hadoop.mapred.JobClient=INFO +log4j.logger.org.apache.hadoop.mapreduce.Job=INFO +log4j.logger.org.apache.tinkerpop.gremlin.hadoop.process.computer.mapreduce.MapReduceGraphComputer=INFO +log4j.logger.org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph=INFO +log4j.logger.org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer=INFO +log4j.logger.org.apache.spark.metrics.MetricsSystem=ERROR +log4j.logger.com.jcabi.manifests.Manifests=OFF diff --git a/scripts/ci/docker-compose/gremlin/logback.xml b/scripts/ci/docker-compose/gremlin/logback.xml new file mode 100644 index 0000000000000..9b85d870a5225 --- /dev/null +++ b/scripts/ci/docker-compose/gremlin/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/scripts/ci/docker-compose/integration-gremlin.yml b/scripts/ci/docker-compose/integration-gremlin.yml new file mode 100644 index 0000000000000..27e9783999afd --- /dev/null +++ b/scripts/ci/docker-compose/integration-gremlin.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +services: + gremlin: + hostname: gremlin + container_name: gremlin + image: tinkerpop/gremlin-server:3.4.2 + labels: + breeze.description: "Integration required for gremlin operator and hook." + volumes: + - ./gremlin:/opt/gremlin-server/conf + - graph-data:/opt/gremlin-server/data + - ./gremlin/gremlin-entrypoint.sh:/opt/gremlin-server/gremlin-entrypoint.sh # New entrypoint script + ports: + - "${GREMLIN_HOST_PORT}:8182" + entrypoint: /opt/gremlin-server/gremlin-entrypoint.sh # Use custom entrypoint + user: "0:0" # Run as root + airflow: + depends_on: + - gremlin + environment: + - INTEGRATION_GREMLIN=true + stdin_open: true +volumes: + graph-data: diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index c9d500209e772..c12a77f7d4927 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -44,6 +44,7 @@ services: - ../../../empty:/opt/airflow/providers/apache/pig/src - ../../../empty:/opt/airflow/providers/apache/pinot/src - ../../../empty:/opt/airflow/providers/apache/spark/src + - ../../../empty:/opt/airflow/providers/apache/tinkerpop/src - ../../../empty:/opt/airflow/providers/apprise/src - ../../../empty:/opt/airflow/providers/arangodb/src - ../../../empty:/opt/airflow/providers/asana/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index c8ccc1f629c46..d94a66ad1c796 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -56,6 +56,7 @@ services: - ../../../providers/apache/pig/tests:/opt/airflow/providers/apache/pig/tests - ../../../providers/apache/pinot/tests:/opt/airflow/providers/apache/pinot/tests - ../../../providers/apache/spark/tests:/opt/airflow/providers/apache/spark/tests + - ../../../providers/apache/tinkerpop/tests:/opt/airflow/providers/apache/tinkerpop/tests - ../../../providers/apprise/tests:/opt/airflow/providers/apprise/tests - ../../../providers/arangodb/tests:/opt/airflow/providers/arangodb/tests - ../../../providers/asana/tests:/opt/airflow/providers/asana/tests diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh index 49c5e7a443ab9..b6ee390577a49 100755 --- a/scripts/in_container/check_environment.sh +++ b/scripts/in_container/check_environment.sh @@ -174,6 +174,10 @@ if [[ ${INTEGRATION_YDB} == "true" ]]; then check_service "YDB Cluster" "run_nc ydb 2136" 50 fi +if [[ ${INTEGRATION_GREMLIN} == "true" ]]; then + check_service "gremlin" "run_nc gremlin 8182" 50 +fi + if [[ ${EXIT_CODE} != 0 ]]; then echo echo "Error: some of the CI environment failed to initialize!"