BUG: fixed missing file handling for remote polling (#55)

MetOffice · Sep 3, 2024 · 09404c3 · 09404c3
1 parent 1b07e24
commit 09404c3
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 21 deletions.
diff --git a/dagrunner/plugin_framework.py b/dagrunner/plugin_framework.py
@@ -203,17 +203,16 @@ def __call__(
 
             if host:
                 # bash equivalent to python glob (glob on remote host)
-                expanded_paths = (
-                    subprocess.run(
-                        f"ssh {host} \"printf '%s\n' {pattern} | grep -v '*'\" || true",
-                        shell=True,
-                        check=True,
-                        text=True,
-                        capture_output=True,
-                    )
-                    .stdout.strip()
-                    .split("\n")
-                )
+                expanded_paths = subprocess.run(
+                    f'ssh {host} \'for file in {pattern}; do if [ -e "$file" ]; then '
+                    'echo "$file"; fi; done\'',
+                    shell=True,
+                    check=True,
+                    text=True,
+                    capture_output=True,
+                ).stdout.strip()
+                if expanded_paths:
+                    expanded_paths = expanded_paths.split("\n")
             else:
                 expanded_paths = glob(pattern)
             if expanded_paths:

diff --git a/dagrunner/tests/plugin_framework/test_DataPolling.py b/dagrunner/tests/plugin_framework/test_DataPolling.py
@@ -56,3 +56,17 @@ def test_specified_host(tmp_file, capsys):
         call_dp(host_tmp_file)
     captured = capsys.readouterr()
     assert f"The following files were polled and found: ['{tmp_file}']" in captured.out
+
+
+def test_specified_host_missing_file(capsys):
+    """<host>:<filepath>"""
+    filepath = "/dummy/file/path.dat"
+    host_tmp_file = f"{socket.gethostname()}:{filepath}"
+    # Mocking gethostname() so that our host doesn't match against our local host check
+    # internally.
+    msg = f"Timeout waiting for: '{filepath}'"
+    with patch(
+        "dagrunner.utils.socket.gethostname", return_value="dummy_host.dummy_domain"
+    ):
+        with pytest.raises(FileNotFoundError, match=msg):
+            call_dp(host_tmp_file, verbose=False)
diff --git a/docs/dagrunner.plugin_framework.md b/docs/dagrunner.plugin_framework.md
@@ -50,7 +50,7 @@ Raises:
 
 ## class: `Input`
 
-[Source](../dagrunner/plugin_framework.py#L241)
+[Source](../dagrunner/plugin_framework.py#L240)
 
 ### Call Signature:
 
@@ -64,7 +64,7 @@ that are 'node aware'.
 
 ### function: `__call__`
 
-[Source](../dagrunner/plugin_framework.py#L242)
+[Source](../dagrunner/plugin_framework.py#L241)
 
 #### Call Signature:
 
@@ -144,7 +144,7 @@ Raises:
 
 ## class: `LoadJson`
 
-[Source](../dagrunner/plugin_framework.py#L275)
+[Source](../dagrunner/plugin_framework.py#L274)
 
 ### Call Signature:
 
@@ -175,7 +175,7 @@ Args:
 
 ### function: `load`
 
-[Source](../dagrunner/plugin_framework.py#L278)
+[Source](../dagrunner/plugin_framework.py#L277)
 
 #### Call Signature:
 
@@ -197,7 +197,7 @@ Raises:
 
 ## class: `LoadPickle`
 
-[Source](../dagrunner/plugin_framework.py#L314)
+[Source](../dagrunner/plugin_framework.py#L313)
 
 ### Call Signature:
 
@@ -228,7 +228,7 @@ Args:
 
 ### function: `load`
 
-[Source](../dagrunner/plugin_framework.py#L317)
+[Source](../dagrunner/plugin_framework.py#L316)
 
 #### Call Signature:
 
@@ -320,7 +320,7 @@ Returns:
 
 ## class: `SaveJson`
 
-[Source](../dagrunner/plugin_framework.py#L286)
+[Source](../dagrunner/plugin_framework.py#L285)
 
 ### Call Signature:
 
@@ -334,7 +334,7 @@ that are 'node aware'.
 
 ### function: `__call__`
 
-[Source](../dagrunner/plugin_framework.py#L287)
+[Source](../dagrunner/plugin_framework.py#L286)
 
 #### Call Signature:
 
@@ -359,7 +359,7 @@ Returns:
 
 ## class: `SavePickle`
 
-[Source](../dagrunner/plugin_framework.py#L325)
+[Source](../dagrunner/plugin_framework.py#L324)
 
 ### Call Signature:
 
@@ -373,7 +373,7 @@ that are 'node aware'.
 
 ### function: `__call__`
 
-[Source](../dagrunner/plugin_framework.py#L326)
+[Source](../dagrunner/plugin_framework.py#L325)
 
 #### Call Signature: