From a7ac50fd4a18e41282d12ee8f28656c6c26b96e9 Mon Sep 17 00:00:00 2001 From: Jay Qi Date: Fri, 2 Apr 2021 18:08:30 -0700 Subject: [PATCH 1/7] Raise exception when open with write mode in call stack --- cloudpathlib/cloudpath.py | 16 ++++++++++++++++ cloudpathlib/exceptions.py | 4 ++++ tests/test_cloudpath_file_io.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 29559d1a..d492313e 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -4,11 +4,14 @@ import fnmatch import os from pathlib import Path, PosixPath, PurePosixPath, WindowsPath +import inspect +import traceback from typing import Any, IO, Iterable, Optional, TYPE_CHECKING, Union from urllib.parse import urlparse from warnings import warn from .exceptions import ( + BuiltInOpenWriteError, ClientMismatchError, CloudPathFileExistsError, CloudPathIsADirectoryError, @@ -205,6 +208,19 @@ def __eq__(self, other: Any) -> bool: return isinstance(other, type(self)) and str(self) == str(other) def __fspath__(self): + WRITE_MODES = {"r+", "w", "w+", "a", "a+", "rb+", "wb", "wb+", "ab", "ab+"} + for frame, _ in traceback.walk_stack(None): + if "open" in frame.f_code.co_names: + code_context = inspect.getframeinfo(frame).code_context + if "open(" in code_context[0]: + for mode in WRITE_MODES: + if f'"{mode}"' in code_context[0] or f"'{mode}'" in code_context[0]: + raise BuiltInOpenWriteError( + "Detected the use of built-in open function with a cloud path and write mode. " + "Cloud paths do not support the open function in write mode; " + "please use the .open() method instead." + ) + if self.is_file(): self._refresh_cache(force_overwrite_from_cloud=False) return str(self._local) diff --git a/cloudpathlib/exceptions.py b/cloudpathlib/exceptions.py index b27b8aab..aa1fd9cd 100644 --- a/cloudpathlib/exceptions.py +++ b/cloudpathlib/exceptions.py @@ -12,6 +12,10 @@ class AnyPathTypeError(CloudPathException, TypeError): pass +class BuiltInOpenWriteError(CloudPathException): + pass + + class ClientMismatchError(CloudPathException, ValueError): pass diff --git a/tests/test_cloudpath_file_io.py b/tests/test_cloudpath_file_io.py index eeb7781c..43c31733 100644 --- a/tests/test_cloudpath_file_io.py +++ b/tests/test_cloudpath_file_io.py @@ -5,7 +5,11 @@ import pytest -from cloudpathlib.exceptions import CloudPathIsADirectoryError, DirectoryNotEmptyError +from cloudpathlib.exceptions import ( + BuiltInOpenWriteError, + CloudPathIsADirectoryError, + DirectoryNotEmptyError, +) def test_file_discovery(rig): @@ -113,3 +117,26 @@ def test_os_open(rig): p = rig.create_cloud_path("dir_0/file0_0.txt") with open(p, "r") as f: assert f.readable() + + with pytest.raises(BuiltInOpenWriteError): + with open(p, "w") as f: + pass + + with pytest.raises(BuiltInOpenWriteError): + with open(p, "wb") as f: + pass + + with pytest.raises(BuiltInOpenWriteError): + with open(p, "a") as f: + pass + + with pytest.raises(BuiltInOpenWriteError): + with open(p, "r+") as f: + pass + + # with pytest.raises(BuiltInOpenWriteError): + # with open( + # p, + # "w", + # ) as f: + # pass From 81e92d3f2f63c6e180a8de66d9b4625b804feb7f Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Sun, 11 Apr 2021 23:46:02 -0700 Subject: [PATCH 2/7] Implement AST-based writeable open detection --- cloudpathlib/cloudpath.py | 95 ++++++++++++++++++++++++++++----- tests/test_cloudpath_file_io.py | 45 +++++++++++----- 2 files changed, 114 insertions(+), 26 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index d492313e..c3ad445e 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -1,11 +1,12 @@ import abc +import ast from collections import defaultdict import collections.abc import fnmatch +import inspect import os from pathlib import Path, PosixPath, PurePosixPath, WindowsPath -import inspect -import traceback +from textwrap import dedent from typing import Any, IO, Iterable, Optional, TYPE_CHECKING, Union from urllib.parse import urlparse from warnings import warn @@ -30,6 +31,8 @@ if TYPE_CHECKING: from .client import Client +CHECK_UNSAFE_OPEN = not (os.getenv("CLOUDPATHLIB_CHECK_UNSAFE_OPEN", "True").lower() == "false") + class CloudImplementation: def __init__(self): @@ -208,18 +211,43 @@ def __eq__(self, other: Any) -> bool: return isinstance(other, type(self)) and str(self) == str(other) def __fspath__(self): - WRITE_MODES = {"r+", "w", "w+", "a", "a+", "rb+", "wb", "wb+", "ab", "ab+"} - for frame, _ in traceback.walk_stack(None): - if "open" in frame.f_code.co_names: - code_context = inspect.getframeinfo(frame).code_context - if "open(" in code_context[0]: - for mode in WRITE_MODES: - if f'"{mode}"' in code_context[0] or f"'{mode}'" in code_context[0]: - raise BuiltInOpenWriteError( - "Detected the use of built-in open function with a cloud path and write mode. " - "Cloud paths do not support the open function in write mode; " - "please use the .open() method instead." - ) + # make sure that we're not getting called by the builtin open + # in a write mode, since we won't actually write to the cloud in + # that scenario + if CHECK_UNSAFE_OPEN: + frame = inspect.currentframe().f_back + + # use entire file source if possible since we need + # a valid ast. + # caller_src_file = Path(inspect.getsourcefile(frame)) + + # if caller_src_file.exists(): + # caller_src = caller_src_file.read_text() + # else: + caller_src = inspect.getsource(frame) + + # also get local variables in the frame + caller_local_variables = frame.f_locals + + # get all the instances in the previous frame of our class + instances_of_type = [ + varname + for varname, instance in caller_local_variables.items() + if isinstance(instance, type(self)) + ] + + # Walk the AST of the previous frame source and see if + # open is called with a variable of our type... + if any( + _is_open_call_write_with_var(n, var_names=instances_of_type, var_type=type(self)) + for n in ast.walk(ast.parse(dedent(caller_src))) + ): + raise BuiltInOpenWriteError( + "Detected the use of built-in open function with a cloud path and write mode. " + "Cloud paths do not support the open function in write mode; " + "please use the .open() method instead.\n\n" + "Line may be incorrect, but call is within file " + ) if self.is_file(): self._refresh_cache(force_overwrite_from_cloud=False) @@ -765,3 +793,42 @@ def _resolve(path: PurePosixPath) -> str: newpath = newpath + sep + name return newpath or sep + + +# This function is used to check if our `__fspath__` implementation has been +# called in a writeable mode from the built-in open function. +def _is_open_call_write_with_var(ast_node, var_names=None, var_type=None): + """For a given AST node, check that the node is a `Call`, and that the + call is to a function with the name `open`, and that the last argument + + If passed, return True if the first argument is a variable with a name in var_names. + + If passed, return True if the first arg is a Call to instantiate var_type. + """ + if not isinstance(ast_node, ast.Call): + return False + if not hasattr(ast_node, "func"): + return False + if not hasattr(ast_node.func, "id"): + return False + if ast_node.func.id != "open": + return False + + # we are in an open call, get the path as first arg + path = ast_node.args[0] + + # get the mode as second arg or kwarg where arg==mode + mode = ( + ast_node.args[1] + if len(ast_node.args) >= 2 + else [kwarg for kwarg in ast_node.keywords if kwarg.arg == "mode"][0].value + ) + + # Ensure the path is either a call to instantiate var_type or + # the name of a variable we know is of the right type + path_is_of_type = (isinstance(path, ast.Call) and path.func.id == var_type.__name__) or ( + hasattr(path, "id") and (path.id in var_names) + ) + + WRITE_MODES = {"r+", "w", "w+", "a", "a+", "rb+", "wb", "wb+", "ab", "ab+"} + return (mode.s in WRITE_MODES) and path_is_of_type diff --git a/tests/test_cloudpath_file_io.py b/tests/test_cloudpath_file_io.py index 43c31733..893f8289 100644 --- a/tests/test_cloudpath_file_io.py +++ b/tests/test_cloudpath_file_io.py @@ -113,30 +113,51 @@ def test_fspath(rig): assert os.fspath(p) == p.fspath -def test_os_open(rig): +def test_os_open_read(rig): p = rig.create_cloud_path("dir_0/file0_0.txt") with open(p, "r") as f: assert f.readable() + +# entire function is passed as source, so check separately +# that all of the built in open write modes fail +def test_os_open_write0(rig): + p = rig.create_cloud_path("dir_0/file0_0.txt") + with pytest.raises(BuiltInOpenWriteError): with open(p, "w") as f: - pass + assert f.readable() + + +def test_os_open_write1(rig): + p = rig.create_cloud_path("dir_0/file0_0.txt") with pytest.raises(BuiltInOpenWriteError): with open(p, "wb") as f: - pass + assert f.readable() + + +def test_os_open_write2(rig): + p = rig.create_cloud_path("dir_0/file0_0.txt") with pytest.raises(BuiltInOpenWriteError): with open(p, "a") as f: - pass + assert f.readable() + + +def test_os_open_write3(rig): + p = rig.create_cloud_path("dir_0/file0_0.txt") with pytest.raises(BuiltInOpenWriteError): with open(p, "r+") as f: - pass - - # with pytest.raises(BuiltInOpenWriteError): - # with open( - # p, - # "w", - # ) as f: - # pass + assert f.readable() + + +def test_os_open_write4(rig): + p = rig.create_cloud_path("dir_0/file0_0.txt") + with pytest.raises(BuiltInOpenWriteError): + with open( + p, + "w", + ) as f: + assert f.readable() From 5b135a901f4b2a11347b7b398808580ed1287d00 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Sun, 11 Apr 2021 23:46:08 -0700 Subject: [PATCH 3/7] docs update --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index d436904c..a21f573c 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,21 @@ Most methods and properties from `pathlib.Path` are supported except for the one | `key` | ❌ | ✅ | ❌ | | `md5` | ✅ | ❌ | ❌ | + +## Writing to cloud files + +**Warning:** You can't call `open(CloudPath("s3://path"), "w")` and have write to the cloud file (reading works fine with the built-in open). Instead of using the Python built-in open, you must use `CloudPath("s3://path").open("w")`. For more iformation, see [#128](https://github.com/drivendataorg/cloudpathlib/issues/128) and [#140](https://github.com/drivendataorg/cloudpathlib/pull/140). + +We try to detect this scenario and raise a `BuiltInOpenWriteError` exception for you. There is a slight performance hit for this check, and if _you are sure_ that either (1) you are not writing to cloud files or (2) you are writing, but you are using the `CloudPath.open` method every time, you can skip this check by setting the environment variable `CLOUDPATHLIB_CHECK_UNSAFE_OPEN=False`. + +If you are passing the `CloudPath` into another library and you see `BuiltInOpenWriteError`, try opening and passing the buffer into that function instead: + +```python +with CloudPath("s3://bucket/path_to_write.txt").open("w") as fp: + function_that_writes(fp) +``` + + ---- Icon made by srip from www.flaticon.com. From ee69c9b7ed07fe9562566913b40e4bf615c048b6 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Mon, 12 Apr 2021 11:11:26 -0700 Subject: [PATCH 4/7] Remove extra code and exception update message --- cloudpathlib/cloudpath.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index c3ad445e..2e95321d 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -217,13 +217,6 @@ def __fspath__(self): if CHECK_UNSAFE_OPEN: frame = inspect.currentframe().f_back - # use entire file source if possible since we need - # a valid ast. - # caller_src_file = Path(inspect.getsourcefile(frame)) - - # if caller_src_file.exists(): - # caller_src = caller_src_file.read_text() - # else: caller_src = inspect.getsource(frame) # also get local variables in the frame @@ -246,7 +239,8 @@ def __fspath__(self): "Detected the use of built-in open function with a cloud path and write mode. " "Cloud paths do not support the open function in write mode; " "please use the .open() method instead.\n\n" - "Line may be incorrect, but call is within file " + "NOTE: Exact line of this error may be incorrect, but open-for-write call is " + "within the same scope. (Skip this check with env var CLOUDPATHLIB_CHECK_UNSAFE_OPEN=False)." ) if self.is_file(): From 35edbb01b46f8acb893a7dd892d508745097369f Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 14 Apr 2021 10:26:47 -0700 Subject: [PATCH 5/7] Update to track and use line numbers for proper error --- cloudpathlib/client.py | 2 +- cloudpathlib/cloudpath.py | 77 ++++++++++++++++++--------------- tests/test_cloudpath_file_io.py | 39 ++++++++++++----- 3 files changed, 71 insertions(+), 47 deletions(-) diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index 544e34ee..226d2550 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -37,7 +37,7 @@ def __init__(self, local_cache_dir: Optional[Union[str, os.PathLike]] = None): def __del__(self) -> None: # make sure temp is cleaned up if we created it - if self._cache_tmp_dir is not None: + if hasattr(self, "_cache_tmp_dir") and self._cache_tmp_dir is not None: self._cache_tmp_dir.cleanup() @classmethod diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 2e95321d..f16c3c92 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -31,7 +31,10 @@ if TYPE_CHECKING: from .client import Client -CHECK_UNSAFE_OPEN = not (os.getenv("CLOUDPATHLIB_CHECK_UNSAFE_OPEN", "True").lower() == "false") +CHECK_UNSAFE_OPEN = str(os.getenv("CLOUDPATHLIB_CHECK_UNSAFE_OPEN", "True").lower()) not in { + "false", + "0", +} class CloudImplementation: @@ -176,7 +179,7 @@ def __init__(self, cloud_path: Union[str, "CloudPath"], client: Optional["Client def __del__(self): # make sure that file handle to local path is closed - if self._handle is not None: + if hasattr(self, "_handle") and self._handle is not None: self._handle.close() @property @@ -217,30 +220,36 @@ def __fspath__(self): if CHECK_UNSAFE_OPEN: frame = inspect.currentframe().f_back - caller_src = inspect.getsource(frame) + # line number of the call for this frame + lineno = inspect.getframeinfo(frame).lineno - # also get local variables in the frame - caller_local_variables = frame.f_locals + # get source lines and start of the entire function + lines, start_lineno = inspect.getsourcelines(frame) - # get all the instances in the previous frame of our class - instances_of_type = [ - varname - for varname, instance in caller_local_variables.items() - if isinstance(instance, type(self)) - ] + # in some contexts, start_lineno is 0, but should be 1-indexed + if start_lineno == 0: + start_lineno = 1 + + # walk forward from this call until we find the line + # that actually has "open" call on it + if "open" not in lines[lineno - start_lineno]: + lineno += 1 + + # 1-indexed line within this scope + line_to_check = (lineno - start_lineno) + 1 - # Walk the AST of the previous frame source and see if - # open is called with a variable of our type... + # Walk the AST of the previous frame source and see if we + # ended up here from a call to the builtin open with and a writeable mode if any( - _is_open_call_write_with_var(n, var_names=instances_of_type, var_type=type(self)) - for n in ast.walk(ast.parse(dedent(caller_src))) + _is_open_call_write_with_var(n, line_to_check) + for n in ast.walk(ast.parse(dedent("".join(lines)))) ): raise BuiltInOpenWriteError( - "Detected the use of built-in open function with a cloud path and write mode. " - "Cloud paths do not support the open function in write mode; " - "please use the .open() method instead.\n\n" - "NOTE: Exact line of this error may be incorrect, but open-for-write call is " - "within the same scope. (Skip this check with env var CLOUDPATHLIB_CHECK_UNSAFE_OPEN=False)." + "Cannot use built-in open function with a CloudPath in a writeable mode. " + "Changes would not be uploaded to the cloud; instead, " + "please use the .open() method instead. " + "NOTE: If you are sure and want to skip this check with " + "set the env var CLOUDPATHLIB_CHECK_UNSAFE_OPEN=False" ) if self.is_file(): @@ -789,15 +798,15 @@ def _resolve(path: PurePosixPath) -> str: return newpath or sep +WRITE_MODES = {"r+", "w", "w+", "a", "a+", "rb+", "wb", "wb+", "ab", "ab+"} + + # This function is used to check if our `__fspath__` implementation has been # called in a writeable mode from the built-in open function. -def _is_open_call_write_with_var(ast_node, var_names=None, var_type=None): +def _is_open_call_write_with_var(ast_node, lineno): """For a given AST node, check that the node is a `Call`, and that the - call is to a function with the name `open`, and that the last argument - - If passed, return True if the first argument is a variable with a name in var_names. - - If passed, return True if the first arg is a Call to instantiate var_type. + call is to a function with the name `open` at line number `lineno`, + and that the last argument or the `mode` kwarg is one of the writeable modes. """ if not isinstance(ast_node, ast.Call): return False @@ -808,8 +817,11 @@ def _is_open_call_write_with_var(ast_node, var_names=None, var_type=None): if ast_node.func.id != "open": return False - # we are in an open call, get the path as first arg - path = ast_node.args[0] + # there may be an invalid open call in the scope, + # but it is not on the line for our current stack, + # so we skip it for now since it will get parsed later + if ast_node.func.lineno != lineno: + return False # get the mode as second arg or kwarg where arg==mode mode = ( @@ -818,11 +830,4 @@ def _is_open_call_write_with_var(ast_node, var_names=None, var_type=None): else [kwarg for kwarg in ast_node.keywords if kwarg.arg == "mode"][0].value ) - # Ensure the path is either a call to instantiate var_type or - # the name of a variable we know is of the right type - path_is_of_type = (isinstance(path, ast.Call) and path.func.id == var_type.__name__) or ( - hasattr(path, "id") and (path.id in var_names) - ) - - WRITE_MODES = {"r+", "w", "w+", "a", "a+", "rb+", "wb", "wb+", "ab", "ab+"} - return (mode.s in WRITE_MODES) and path_is_of_type + return mode.s.lower() in WRITE_MODES diff --git a/tests/test_cloudpath_file_io.py b/tests/test_cloudpath_file_io.py index 893f8289..917e52b3 100644 --- a/tests/test_cloudpath_file_io.py +++ b/tests/test_cloudpath_file_io.py @@ -121,16 +121,19 @@ def test_os_open_read(rig): # entire function is passed as source, so check separately # that all of the built in open write modes fail -def test_os_open_write0(rig): +def test_os_open_write1(rig): p = rig.create_cloud_path("dir_0/file0_0.txt") + with open(p, "r") as f: + assert f.readable() + with pytest.raises(BuiltInOpenWriteError): with open(p, "w") as f: assert f.readable() - -def test_os_open_write1(rig): - p = rig.create_cloud_path("dir_0/file0_0.txt") + with pytest.raises(BuiltInOpenWriteError): + with open(p, "W") as f: + assert f.readable() with pytest.raises(BuiltInOpenWriteError): with open(p, "wb") as f: @@ -144,20 +147,36 @@ def test_os_open_write2(rig): with open(p, "a") as f: assert f.readable() - -def test_os_open_write3(rig): - p = rig.create_cloud_path("dir_0/file0_0.txt") - with pytest.raises(BuiltInOpenWriteError): - with open(p, "r+") as f: + with open(rig.create_cloud_path("dir_0/file0_0.txt"), "r+") as f: assert f.readable() -def test_os_open_write4(rig): +def test_os_open_write3(rig): p = rig.create_cloud_path("dir_0/file0_0.txt") + # first call should not raise even though there is an unsafe open in same scope + with open( + p, + "r", + ) as f: + assert f.readable() + with pytest.raises(BuiltInOpenWriteError): with open( p, "w", ) as f: assert f.readable() + + +def test_os_open_write4(monkeypatch, rig): + p = rig.create_cloud_path("dir_0/file0_0.txt") + + monkeypatch.setattr("cloudpathlib.cloudpath.CHECK_UNSAFE_OPEN", False) + + # unsafe write check is skipped + with open( + p, + "w+", + ) as f: + assert f.readable() From 6395eb5b80bda1e21e2b337381e3c58995349006 Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Wed, 14 Apr 2021 10:32:13 -0700 Subject: [PATCH 6/7] Use writable assert for proper error message --- tests/test_cloudpath_file_io.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_cloudpath_file_io.py b/tests/test_cloudpath_file_io.py index 917e52b3..03cdf6aa 100644 --- a/tests/test_cloudpath_file_io.py +++ b/tests/test_cloudpath_file_io.py @@ -129,15 +129,15 @@ def test_os_open_write1(rig): with pytest.raises(BuiltInOpenWriteError): with open(p, "w") as f: - assert f.readable() + assert f.writable() with pytest.raises(BuiltInOpenWriteError): with open(p, "W") as f: - assert f.readable() + assert f.writable() with pytest.raises(BuiltInOpenWriteError): with open(p, "wb") as f: - assert f.readable() + assert f.writable() def test_os_open_write2(rig): @@ -145,7 +145,7 @@ def test_os_open_write2(rig): with pytest.raises(BuiltInOpenWriteError): with open(p, "a") as f: - assert f.readable() + assert f.writable() with pytest.raises(BuiltInOpenWriteError): with open(rig.create_cloud_path("dir_0/file0_0.txt"), "r+") as f: @@ -166,7 +166,7 @@ def test_os_open_write3(rig): p, "w", ) as f: - assert f.readable() + assert f.writable() def test_os_open_write4(monkeypatch, rig): From e6c2bae14bb733680b799aa3188be85495d081bf Mon Sep 17 00:00:00 2001 From: Peter Bull Date: Fri, 14 May 2021 13:00:33 -0700 Subject: [PATCH 7/7] Fix line mix-up in python<=3.7 --- cloudpathlib/cloudpath.py | 48 ++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index f16c3c92..df421f8a 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -6,6 +6,7 @@ import inspect import os from pathlib import Path, PosixPath, PurePosixPath, WindowsPath +import sys from textwrap import dedent from typing import Any, IO, Iterable, Optional, TYPE_CHECKING, Union from urllib.parse import urlparse @@ -221,36 +222,41 @@ def __fspath__(self): frame = inspect.currentframe().f_back # line number of the call for this frame - lineno = inspect.getframeinfo(frame).lineno + lineno = frame.f_lineno # get source lines and start of the entire function lines, start_lineno = inspect.getsourcelines(frame) - # in some contexts, start_lineno is 0, but should be 1-indexed + # in some contexts like jupyter, start_lineno is 0, but should be 1-indexed if start_lineno == 0: start_lineno = 1 - # walk forward from this call until we find the line - # that actually has "open" call on it - if "open" not in lines[lineno - start_lineno]: - lineno += 1 + all_lines = "".join(lines) - # 1-indexed line within this scope - line_to_check = (lineno - start_lineno) + 1 + if "open" in all_lines: + # walk from this call until we find the line + # that actually has "open" call on it + # only needed on Python <= 3.7 + if (sys.version_info.major, sys.version_info.minor) <= (3, 7): + while "open" not in lines[lineno - start_lineno]: + lineno -= 1 - # Walk the AST of the previous frame source and see if we - # ended up here from a call to the builtin open with and a writeable mode - if any( - _is_open_call_write_with_var(n, line_to_check) - for n in ast.walk(ast.parse(dedent("".join(lines)))) - ): - raise BuiltInOpenWriteError( - "Cannot use built-in open function with a CloudPath in a writeable mode. " - "Changes would not be uploaded to the cloud; instead, " - "please use the .open() method instead. " - "NOTE: If you are sure and want to skip this check with " - "set the env var CLOUDPATHLIB_CHECK_UNSAFE_OPEN=False" - ) + # 1-indexed line within this scope + line_to_check = (lineno - start_lineno) + 1 + + # Walk the AST of the previous frame source and see if we + # ended up here from a call to the builtin open with and a writeable mode + if any( + _is_open_call_write_with_var(n, line_to_check) + for n in ast.walk(ast.parse(dedent(all_lines))) + ): + raise BuiltInOpenWriteError( + "Cannot use built-in open function with a CloudPath in a writeable mode. " + "Changes would not be uploaded to the cloud; instead, " + "please use the .open() method instead. " + "NOTE: If you are sure and want to skip this check with " + "set the env var CLOUDPATHLIB_CHECK_UNSAFE_OPEN=False" + ) if self.is_file(): self._refresh_cache(force_overwrite_from_cloud=False)