From 7a85c591d6ac04f154bc070e11a17150043f1e79 Mon Sep 17 00:00:00 2001 From: Vincent Beck Date: Mon, 2 May 2022 13:42:20 -0600 Subject: [PATCH 1/2] Add sample dag and doc for S3ListOperator --- .../amazon/aws/example_dags/example_s3.py | 10 ++++++++++ airflow/providers/amazon/aws/operators/s3.py | 4 ++++ .../operators/s3.rst | 15 +++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/airflow/providers/amazon/aws/example_dags/example_s3.py b/airflow/providers/amazon/aws/example_dags/example_s3.py index ecd9d374cf688..9627703c0bb87 100644 --- a/airflow/providers/amazon/aws/example_dags/example_s3.py +++ b/airflow/providers/amazon/aws/example_dags/example_s3.py @@ -30,6 +30,7 @@ S3DeleteObjectsOperator, S3FileTransformOperator, S3GetBucketTaggingOperator, + S3ListOperator, S3PutBucketTaggingOperator, ) from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor, S3KeysUnchangedSensor @@ -115,6 +116,14 @@ def check_fn(files: List) -> bool: ) # [END howto_operator_s3_create_object] + # [START howto_operator_s3_list] + list_keys = S3ListOperator( + task_id="s3_list_operator", + bucket=BUCKET_NAME, + prefix=PREFIX, + ) + # [END howto_operator_s3_list] + # [START howto_sensor_s3_key_single_key] # Check if a file exists s3_sensor_one_key = S3KeySensor( @@ -193,6 +202,7 @@ def check_fn(files: List) -> bool: get_tagging, delete_tagging, s3_create_object, + list_keys, [s3_sensor_one_key, s3_sensor_two_keys, s3_sensor_key_function], s3_copy_object, s3_sensor_keys_unchanged, diff --git a/airflow/providers/amazon/aws/operators/s3.py b/airflow/providers/amazon/aws/operators/s3.py index a33913701d7cc..1e259ffdef124 100644 --- a/airflow/providers/amazon/aws/operators/s3.py +++ b/airflow/providers/amazon/aws/operators/s3.py @@ -618,6 +618,10 @@ class S3ListOperator(BaseOperator): This operator returns a python list with the name of objects which can be used by `xcom` in the downstream task. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:S3ListOperator` + :param bucket: The S3 bucket where to find the objects. (templated) :param prefix: Prefix string to filters the objects whose name begin with such prefix. (templated) diff --git a/docs/apache-airflow-providers-amazon/operators/s3.rst b/docs/apache-airflow-providers-amazon/operators/s3.rst index 00f1fe1143353..0be2d2ae17ad9 100644 --- a/docs/apache-airflow-providers-amazon/operators/s3.rst +++ b/docs/apache-airflow-providers-amazon/operators/s3.rst @@ -196,6 +196,21 @@ To create a new (or replace) Amazon S3 object you can use :start-after: [START howto_operator_s3_create_object] :end-before: [END howto_operator_s3_create_object] +.. _howto/operator:S3ListOperator: + +List Amazon S3 objects +---------------------- + +To list all Amazon S3 objects within an Amazon S3 bucket you can use +:class:`~airflow.providers.amazon.aws.operators.s3.S3ListOperator`. +You can specify ``prefix`` to filter the objects whose name begin with such prefix. + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_s3_list] + :end-before: [END howto_operator_s3_list] + .. _howto/operator:S3CopyObjectOperator: Copy an Amazon S3 object From 81edeca332cce7e4485a0b4be0555733efa36598 Mon Sep 17 00:00:00 2001 From: Vincent <97131062+vincbeck@users.noreply.github.com> Date: Mon, 9 May 2022 09:29:17 -0600 Subject: [PATCH 2/2] Fix doc Co-authored-by: Josh Fell <48934154+josh-fell@users.noreply.github.com> --- docs/apache-airflow-providers-amazon/operators/s3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-amazon/operators/s3.rst b/docs/apache-airflow-providers-amazon/operators/s3.rst index 0be2d2ae17ad9..01222205661ee 100644 --- a/docs/apache-airflow-providers-amazon/operators/s3.rst +++ b/docs/apache-airflow-providers-amazon/operators/s3.rst @@ -203,7 +203,7 @@ List Amazon S3 objects To list all Amazon S3 objects within an Amazon S3 bucket you can use :class:`~airflow.providers.amazon.aws.operators.s3.S3ListOperator`. -You can specify ``prefix`` to filter the objects whose name begin with such prefix. +You can specify a ``prefix`` to filter the objects whose name begins with such prefix. .. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_s3.py :language: python