diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py index d89986188bfa1..2fa5de1812b3c 100644 --- a/airflow-core/tests/unit/always/test_project_structure.py +++ b/airflow-core/tests/unit/always/test_project_structure.py @@ -477,6 +477,7 @@ class TestGoogleProviderProjectStructure(ExampleCoverageTest, AssetsCoverageTest } MISSING_EXAMPLES_FOR_CLASSES = { + "airflow.providers.google.cloud.operators.life_sciences.LifeSciencesRunPipelineOperator", "airflow.providers.google.cloud.operators.dlp.CloudDLPRedactImageOperator", "airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraToGCSOperator", "airflow.providers.google.cloud.transfers.adls_to_gcs.ADLSToGCSOperator", diff --git a/providers/google/docs/operators/cloud/life_sciences.rst b/providers/google/docs/operators/cloud/life_sciences.rst index dab38193b7269..6f21907b17789 100644 --- a/providers/google/docs/operators/cloud/life_sciences.rst +++ b/providers/google/docs/operators/cloud/life_sciences.rst @@ -23,34 +23,14 @@ The `Google Cloud Life Sciences `__ is series of compute engine containers on the Google Cloud. It is used to process, analyze and annotate genomics and biomedical data at scale. +.. warning:: + The Cloud Life Sciences will be discontinued on July 8, 2025. Please, use Google Cloud Batch instead. + Prerequisite Tasks ^^^^^^^^^^^^^^^^^^ .. include:: /operators/_partials/prerequisite_tasks.rst - -Pipeline Configuration -^^^^^^^^^^^^^^^^^^^^^^ -In order to run the pipeline, it is necessary to configure the request body. -Here is an example of the pipeline configuration with a single action. - -.. exampleinclude:: /../../google/tests/system/google/cloud/life_sciences/example_life_sciences.py - :language: python - :dedent: 0 - :start-after: [START howto_configure_simple_action_pipeline] - :end-before: [END howto_configure_simple_action_pipeline] - -The pipeline can also be configured with multiple action. - -.. exampleinclude:: /../../google/tests/system/google/cloud/life_sciences/example_life_sciences.py - :language: python - :dedent: 0 - :start-after: [START howto_configure_multiple_action_pipeline] - :end-before: [END howto_configure_multiple_action_pipeline] - -Read about the `request body parameters `__ -to understand all the fields you can include in the configuration - .. _howto/operator:LifeSciencesRunPipelineOperator: Running a pipeline @@ -59,11 +39,15 @@ Use the :class:`~airflow.providers.google.cloud.operators.life_sciences.LifeSciencesRunPipelineOperator` to execute pipelines. -.. exampleinclude:: /../../google/tests/system/google/cloud/life_sciences/example_life_sciences.py +This operator is deprecated and will be removed after July 08, 2025. +All the functionality and new features are available on the Google Cloud Batch platform. Please use +:class:`~airflow.providers.google.cloud.operators.CloudBatchSubmitJobOperator` + +.. exampleinclude:: /../../google/tests/system/google/cloud/cloud_batch/example_cloud_batch.py :language: python :dedent: 0 - :start-after: [START howto_run_pipeline] - :end-before: [END howto_run_pipeline] + :start-after: [START howto_operator_batch_job_creation] + :end-before: [END howto_operator_batch_job_creation] Reference ^^^^^^^^^ diff --git a/providers/google/tests/system/google/cloud/life_sciences/__init__.py b/providers/google/tests/system/google/cloud/life_sciences/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/providers/google/tests/system/google/cloud/life_sciences/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/providers/google/tests/system/google/cloud/life_sciences/example_life_sciences.py b/providers/google/tests/system/google/cloud/life_sciences/example_life_sciences.py deleted file mode 100644 index 02f8d82c5cc69..0000000000000 --- a/providers/google/tests/system/google/cloud/life_sciences/example_life_sciences.py +++ /dev/null @@ -1,144 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import os -from datetime import datetime -from pathlib import Path - -from airflow.models.baseoperator import chain -from airflow.models.dag import DAG -from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator -from airflow.providers.google.cloud.operators.life_sciences import LifeSciencesRunPipelineOperator -from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator -from airflow.utils.trigger_rule import TriggerRule - -from system.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID - -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") -PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID -DAG_ID = "life_sciences" - -BUCKET_NAME = f"bucket_{DAG_ID}-{ENV_ID}" - -FILE_NAME = "file" -LOCATION = "us-central1" - -CURRENT_FOLDER = Path(__file__).parent -FILE_LOCAL_PATH = str(Path(CURRENT_FOLDER) / "resources" / FILE_NAME) - -# [START howto_configure_simple_action_pipeline] -SIMPLE_ACTION_PIPELINE = { - "pipeline": { - "actions": [ - {"imageUri": "bash", "commands": ["-c", "echo Hello, world"]}, - ], - "resources": { - "regions": [f"{LOCATION}"], - "virtualMachine": { - "machineType": "n1-standard-1", - }, - }, - }, -} -# [END howto_configure_simple_action_pipeline] - -# [START howto_configure_multiple_action_pipeline] -MULTI_ACTION_PIPELINE = { - "pipeline": { - "actions": [ - { - "imageUri": "google/cloud-sdk", - "commands": ["gsutil", "cp", f"gs://{BUCKET_NAME}/{FILE_NAME}", "/tmp"], - }, - {"imageUri": "bash", "commands": ["-c", "echo Hello, world"]}, - { - "imageUri": "google/cloud-sdk", - "commands": [ - "gsutil", - "cp", - f"gs://{BUCKET_NAME}/{FILE_NAME}", - f"gs://{BUCKET_NAME}/output.in", - ], - }, - ], - "resources": { - "regions": [f"{LOCATION}"], - "virtualMachine": { - "machineType": "n1-standard-1", - }, - }, - } -} -# [END howto_configure_multiple_action_pipeline] - -with DAG( - DAG_ID, - schedule="@once", - start_date=datetime(2021, 1, 1), - catchup=False, - tags=["example", "life-sciences"], -) as dag: - create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=BUCKET_NAME) - - upload_file = LocalFilesystemToGCSOperator( - task_id="upload_file", - src=FILE_LOCAL_PATH, - dst=FILE_NAME, - bucket=BUCKET_NAME, - ) - - # [START howto_run_pipeline] - simple_life_science_action_pipeline = LifeSciencesRunPipelineOperator( - task_id="simple-action-pipeline", - body=SIMPLE_ACTION_PIPELINE, - project_id=PROJECT_ID, - location=LOCATION, - ) - # [END howto_run_pipeline] - - multiple_life_science_action_pipeline = LifeSciencesRunPipelineOperator( - task_id="multi-action-pipeline", body=MULTI_ACTION_PIPELINE, project_id=PROJECT_ID, location=LOCATION - ) - - delete_bucket = GCSDeleteBucketOperator( - task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE - ) - - chain( - # TEST SETUP - create_bucket, - upload_file, - # TEST BODY - simple_life_science_action_pipeline, - multiple_life_science_action_pipeline, - # TEST TEARDOWN - delete_bucket, - ) - - from tests_common.test_utils.watcher import watcher - - # This test needs watcher in order to properly mark success/failure - # when "tearDown" task with trigger rule is part of the DAG - list(dag.tasks) >> watcher() - - -from tests_common.test_utils.system_tests import get_test_run # noqa: E402 - -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) -test_run = get_test_run(dag) diff --git a/providers/google/tests/system/google/cloud/life_sciences/resources/__init__.py b/providers/google/tests/system/google/cloud/life_sciences/resources/__init__.py deleted file mode 100644 index 13a83393a9124..0000000000000 --- a/providers/google/tests/system/google/cloud/life_sciences/resources/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/providers/google/tests/system/google/cloud/life_sciences/resources/file b/providers/google/tests/system/google/cloud/life_sciences/resources/file deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/providers/google/tests/unit/google/cloud/hooks/test_life_sciences.py b/providers/google/tests/unit/google/cloud/hooks/test_life_sciences.py index ce5404e81b60b..d3e13213600ef 100644 --- a/providers/google/tests/unit/google/cloud/hooks/test_life_sciences.py +++ b/providers/google/tests/unit/google/cloud/hooks/test_life_sciences.py @@ -26,7 +26,7 @@ import pytest -from airflow.exceptions import AirflowException +from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning from airflow.providers.google.cloud.hooks.life_sciences import LifeSciencesHook from unit.google.cloud.utils.base_gcp_mock import ( @@ -51,11 +51,12 @@ class TestLifeSciencesHookWithPassedProjectId: def setup_method(self): - with mock.patch( - "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", - new=mock_base_gcp_hook_default_project_id, - ): - self.hook = LifeSciencesHook(gcp_conn_id="test") + with pytest.warns(AirflowProviderDeprecationWarning): + with mock.patch( + "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", + new=mock_base_gcp_hook_default_project_id, + ): + self.hook = LifeSciencesHook(gcp_conn_id="test") def test_location_path(self): path = "projects/life-science-project-id/locations/test-location" @@ -162,11 +163,12 @@ def test_error_operation(self, _, get_conn_mock, mock_project_id): class TestLifeSciencesHookWithDefaultProjectIdFromConnection: def setup_method(self): - with mock.patch( - "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", - new=mock_base_gcp_hook_default_project_id, - ): - self.hook = LifeSciencesHook(gcp_conn_id="test") + with pytest.warns(AirflowProviderDeprecationWarning): + with mock.patch( + "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", + new=mock_base_gcp_hook_default_project_id, + ): + self.hook = LifeSciencesHook(gcp_conn_id="test") @mock.patch("airflow.providers.google.cloud.hooks.life_sciences.LifeSciencesHook._authorize") @mock.patch("airflow.providers.google.cloud.hooks.life_sciences.build") @@ -268,11 +270,12 @@ def test_error_operation(self, _, get_conn_mock, mock_project_id): class TestLifeSciencesHookWithoutProjectId: def setup_method(self): - with mock.patch( - "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", - new=mock_base_gcp_hook_no_default_project_id, - ): - self.hook = LifeSciencesHook(gcp_conn_id="test") + with pytest.warns(AirflowProviderDeprecationWarning): + with mock.patch( + "airflow.providers.google.common.hooks.base_google.GoogleBaseHook.__init__", + new=mock_base_gcp_hook_no_default_project_id, + ): + self.hook = LifeSciencesHook(gcp_conn_id="test") @mock.patch("airflow.providers.google.cloud.hooks.life_sciences.LifeSciencesHook._authorize") @mock.patch("airflow.providers.google.cloud.hooks.life_sciences.build") diff --git a/providers/google/tests/unit/google/cloud/operators/test_life_sciences.py b/providers/google/tests/unit/google/cloud/operators/test_life_sciences.py index b5b33e58f6fe4..e49c428d1a44d 100644 --- a/providers/google/tests/unit/google/cloud/operators/test_life_sciences.py +++ b/providers/google/tests/unit/google/cloud/operators/test_life_sciences.py @@ -21,6 +21,9 @@ from unittest import mock +import pytest + +from airflow.exceptions import AirflowProviderDeprecationWarning from airflow.providers.google.cloud.operators.life_sciences import LifeSciencesRunPipelineOperator TEST_BODY = {"pipeline": {"actions": [{}], "resources": {}, "environment": {}, "timeout": "3.5s"}} @@ -40,11 +43,12 @@ class TestLifeSciencesRunPipelineOperator: def test_executes(self, mock_hook): mock_instance = mock_hook.return_value mock_instance.run_pipeline.return_value = TEST_OPERATION - operator = LifeSciencesRunPipelineOperator( - task_id="task-id", body=TEST_BODY, location=TEST_LOCATION, project_id=TEST_PROJECT_ID - ) - context = mock.MagicMock() - result = operator.execute(context=context) + with pytest.warns(AirflowProviderDeprecationWarning): + operator = LifeSciencesRunPipelineOperator( + task_id="task-id", body=TEST_BODY, location=TEST_LOCATION, project_id=TEST_PROJECT_ID + ) + context = mock.MagicMock() + result = operator.execute(context=context) assert result == TEST_OPERATION @@ -52,11 +56,12 @@ def test_executes(self, mock_hook): def test_executes_without_project_id(self, mock_hook): mock_instance = mock_hook.return_value mock_instance.run_pipeline.return_value = TEST_OPERATION - operator = LifeSciencesRunPipelineOperator( - task_id="task-id", - body=TEST_BODY, - location=TEST_LOCATION, - ) - context = mock.MagicMock() - result = operator.execute(context=context) + with pytest.warns(AirflowProviderDeprecationWarning): + operator = LifeSciencesRunPipelineOperator( + task_id="task-id", + body=TEST_BODY, + location=TEST_LOCATION, + ) + context = mock.MagicMock() + result = operator.execute(context=context) assert result == TEST_OPERATION