From f42cd6db18a80bb33d115bef5f8055a80c99f706 Mon Sep 17 00:00:00 2001 From: Aaron Schaer Date: Tue, 21 Mar 2023 10:37:43 -0500 Subject: [PATCH 1/3] include filter rules --- .../20230321_152951_aaschaer_include.rst | 10 ++++++ .../services/transfer/data/transfer_data.py | 31 +++++++++++++------ .../mypy-ignore-tests/transfer_data.py | 2 +- 3 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 changelog.d/20230321_152951_aaschaer_include.rst diff --git a/changelog.d/20230321_152951_aaschaer_include.rst b/changelog.d/20230321_152951_aaschaer_include.rst new file mode 100644 index 000000000..e05360182 --- /dev/null +++ b/changelog.d/20230321_152951_aaschaer_include.rst @@ -0,0 +1,10 @@ +.. +.. A new scriv changelog fragment +.. +.. Add one or more items to the list below describing the change in clear, concise terms. +.. +.. Leave the ":pr:`...`" text alone. When you open a pull request, GitHub Actions will +.. automatically replace it when the PR is merged. +.. + +* Support passing "include" as a transfer `filter_rule` method (:pr:`NUMBER`) diff --git a/src/globus_sdk/services/transfer/data/transfer_data.py b/src/globus_sdk/services/transfer/data/transfer_data.py index 15a9be1e0..885477c60 100644 --- a/src/globus_sdk/services/transfer/data/transfer_data.py +++ b/src/globus_sdk/services/transfer/data/transfer_data.py @@ -352,7 +352,7 @@ def add_filter_rule( self, name: str, *, - method: Literal["exclude"] = "exclude", + method: Literal["include", "exclude"] = "exclude", type: None # pylint: disable=redefined-builtin | (Literal["file", "dir"]) = None, ) -> None: @@ -360,31 +360,44 @@ def add_filter_rule( Add a filter rule to the transfer document. These rules specify which items are or are not included when recursively - transferring directories. - - Currently, only ``method="exclude"`` (the default) is supported. This means that - items matching the ``name`` field will be excluded from the transfer. + transferring directories. Each item that is found during recursive directory + traversal is matched against these rules in the order they are listed. + The method of the first filter rule that matches an item is applied (either + "include" or "exclude"), and filter rule matching stops. If no rules match, + the item is included in the transfer. Notably, this makes "include" filter + rules only useful when overriding more general "exclude" filter rules later + in the list. :param name: A pattern to match against item names. Wildcards are supported, as are character groups: ``*`` matches everything, ``?`` matches any single character, ``[]`` matches any single character within the brackets, and ``[!]`` matches any single character not within the brackets. :type name: str - :param method: The method to use for filtering. Only the default, ``"exclude"`` - is supported. + :param method: The method to use for filtering. If "exclude" (the default) + items matching this rule will not be included in the transfer. If "include" + items matching this rule will be included in the transfer. :type method: str, optional :param type: The types of items on which to apply this filter rule. Either ``"file"`` or ``"dir"``. If unspecified, the rule applies to both. + Note that if a ``"dir"`` is excluded then all items within it will + also be excluded regardless if they would have matched any include rules. :type type: str, optional Example Usage: >>> tdata = TransferData(...) - >>> tdata.add_filter_rule("*.tgz", type="file") - >>> tdata.add_filter_rule("*.tar.gz", type="file") + >>> tdata.add_filter_rule(method="exclude", *.tgz", type="file") + >>> tdata.add_filter_rule(method="exclude", *.tar.gz", type="file") ``tdata`` now describes a transfer which will skip any gzipped tar files with the extensions `.tgz` or `.tar.gz` + + >>> tdata = TransferData(...) + >>> tdata.add_filter_rule(method="include", *.txt", type="file") + >>> tdata.add_filter_rule(method="exclude", "*", type="file") + + ``tdata`` now describes a transfer which will only transfer files + with the `.txt` extension. """ if "filter_rules" not in self: self["filter_rules"] = [] diff --git a/tests/non-pytest/mypy-ignore-tests/transfer_data.py b/tests/non-pytest/mypy-ignore-tests/transfer_data.py index a93840550..836df171f 100644 --- a/tests/non-pytest/mypy-ignore-tests/transfer_data.py +++ b/tests/non-pytest/mypy-ignore-tests/transfer_data.py @@ -21,4 +21,4 @@ tdata.add_filter_rule("*.tgz", type="file") # bad values rejected (Literal) tdata.add_filter_rule("*.tgz", type="files") # type: ignore[arg-type] -tdata.add_filter_rule("*.tgz", method="include") # type: ignore[arg-type] +tdata.add_filter_rule("*.tgz", method="unclude") # type: ignore[arg-type] From 1a212eaea6f17df896902c2dbb73b081ca008c75 Mon Sep 17 00:00:00 2001 From: aaschaer Date: Wed, 22 Mar 2023 15:40:26 -0500 Subject: [PATCH 2/3] Fix quotes in example doc Co-authored-by: Kurt McKee --- src/globus_sdk/services/transfer/data/transfer_data.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/globus_sdk/services/transfer/data/transfer_data.py b/src/globus_sdk/services/transfer/data/transfer_data.py index 885477c60..f2443441e 100644 --- a/src/globus_sdk/services/transfer/data/transfer_data.py +++ b/src/globus_sdk/services/transfer/data/transfer_data.py @@ -386,18 +386,18 @@ def add_filter_rule( Example Usage: >>> tdata = TransferData(...) - >>> tdata.add_filter_rule(method="exclude", *.tgz", type="file") - >>> tdata.add_filter_rule(method="exclude", *.tar.gz", type="file") + >>> tdata.add_filter_rule(method="exclude", "*.tgz", type="file") + >>> tdata.add_filter_rule(method="exclude", "*.tar.gz", type="file") ``tdata`` now describes a transfer which will skip any gzipped tar files with - the extensions `.tgz` or `.tar.gz` + the extensions ``.tgz`` or ``.tar.gz`` >>> tdata = TransferData(...) - >>> tdata.add_filter_rule(method="include", *.txt", type="file") + >>> tdata.add_filter_rule(method="include", "*.txt", type="file") >>> tdata.add_filter_rule(method="exclude", "*", type="file") ``tdata`` now describes a transfer which will only transfer files - with the `.txt` extension. + with the ``.txt`` extension. """ if "filter_rules" not in self: self["filter_rules"] = [] From c75c00cd2a2aac8d25c81d1ebdb41d7245a0d2bf Mon Sep 17 00:00:00 2001 From: Aaron Schaer Date: Wed, 29 Mar 2023 16:16:26 -0500 Subject: [PATCH 3/3] fix docstring indentation --- src/globus_sdk/services/transfer/data/transfer_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/globus_sdk/services/transfer/data/transfer_data.py b/src/globus_sdk/services/transfer/data/transfer_data.py index f2443441e..017ba89d8 100644 --- a/src/globus_sdk/services/transfer/data/transfer_data.py +++ b/src/globus_sdk/services/transfer/data/transfer_data.py @@ -374,8 +374,8 @@ def add_filter_rule( ``[!]`` matches any single character not within the brackets. :type name: str :param method: The method to use for filtering. If "exclude" (the default) - items matching this rule will not be included in the transfer. If "include" - items matching this rule will be included in the transfer. + items matching this rule will not be included in the transfer. If + "include" items matching this rule will be included in the transfer. :type method: str, optional :param type: The types of items on which to apply this filter rule. Either ``"file"`` or ``"dir"``. If unspecified, the rule applies to both.