Add doc and sample dag for GCSToS3Operator (#23730)

airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from datetime import datetime
+
+from airflow import DAG
+from airflow.providers.amazon.aws.transfers.gcs_to_s3 import GCSToS3Operator
+
+BUCKET = os.getenv("BUCKET", "bucket")
+S3_KEY = os.getenv("S3_KEY", "s3://<bucket>/<prefix>")
+
+with DAG(
+    dag_id="example_gcs_to_s3",
+    schedule_interval=None,
+    start_date=datetime(2021, 1, 1),
+    tags=["example"],
+    catchup=False,
+) as dag:
+    # [START howto_transfer_gcs_to_s3]
+    gcs_to_s3 = GCSToS3Operator(
+        task_id="gcs_to_s3",
+        bucket=BUCKET,
+        dest_s3_key=S3_KEY,
+        replace=True,
+    )
+    # [END howto_transfer_gcs_to_s3]

airflow/providers/amazon/aws/transfers/gcs_to_s3.py

@@ -32,6 +32,10 @@ class GCSToS3Operator(BaseOperator):
     """
     Synchronizes a Google Cloud Storage bucket with an S3 bucket.
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:GCSToS3Operator`
+
     :param bucket: The Google Cloud Storage bucket to find the objects. (templated)
     :param prefix: Prefix string which filters objects whose name begin with
         this prefix. (templated)

airflow/providers/amazon/provider.yaml

@@ -482,6 +482,7 @@ transfers:
     python-module: airflow.providers.amazon.aws.transfers.dynamodb_to_s3
   - source-integration-name: Google Cloud Storage (GCS)
     target-integration-name: Amazon Simple Storage Service (S3)
+    how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst
     python-module: airflow.providers.amazon.aws.transfers.gcs_to_s3
   - source-integration-name: Amazon Glacier
     target-integration-name: Google Cloud Storage (GCS)

docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst

@@ -0,0 +1,52 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+===================================================
+Google Cloud Storage to Amazon S3 transfer operator
+===================================================
+
+Use the ``GCSToS3Operator`` transfer to copy the data from Google Cloud Storage to Amazon Simple Storage Service (S3).
+
+Prerequisite Tasks
+------------------
+
+.. include:: ../_partials/prerequisite_tasks.rst
+
+Operators
+---------
+
+.. _howto/operator:GCSToS3Operator:
+
+Google Cloud Storage to Amazon S3
+=================================
+
+To copy data from a Google Cloud Storage bucket to an Amazon S3 bucket you can use
+:class:`~airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator`
+
+Example usage:
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_gcs_to_s3.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_transfer_gcs_to_s3]
+    :end-before: [END howto_transfer_gcs_to_s3]
+
+Reference
+---------
+
+* `Google Cloud Storage Client library <https://googleapis.dev/python/storage/latest/client.html>`__
+* `AWS boto3 Library Documentation for Amazon S3 <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html>`__