[Ready for Review] fix bug where client can not access foreach stack

metaflow/__init__.py

@@ -143,6 +143,9 @@ class and related decorators.
     DataArtifact,
 )
 
+# Import data class within tuple_util but not introduce new symbols.
+from . import tuple_util
+
 __version_addl__ = []
 _ext_debug("Loading top-level modules")
 for m in _tl_modules:

metaflow/plugins/datatools/s3/s3.py

@@ -21,14 +21,14 @@
     TEMPDIR,
 )
 from metaflow.util import (
-    namedtuple_with_defaults,
     is_stringish,
     to_bytes,
     to_unicode,
     to_fileobj,
     url_quote,
     url_unquote,
 )
+from metaflow.tuple_util import namedtuple_with_defaults
 from metaflow.exception import MetaflowException
 from metaflow.debug import debug
 import metaflow.tracing as tracing

metaflow/task.py

@@ -23,18 +23,7 @@
 from .clone_util import clone_task_helper
 from .metaflow_current import current
 from metaflow.tracing import get_trace_id
-from metaflow.util import namedtuple_with_defaults
-
-foreach_frame_field_list = [
-    ("step", str),
-    ("var", str),
-    ("num_splits", int),
-    ("index", int),
-    ("value", str),
-]
-ForeachFrame = namedtuple_with_defaults(
-    "ForeachFrame", foreach_frame_field_list, (None,) * (len(foreach_frame_field_list))
-)
+from metaflow.tuple_util import ForeachFrame
 
 # Maximum number of characters of the foreach path that we store in the metadata.
 MAX_FOREACH_PATH_LENGTH = 256

metaflow/tuple_util.py

@@ -0,0 +1,27 @@
+# Keep this file minimum dependency as this will be imported by metaflow at bootup.
+def namedtuple_with_defaults(typename, field_descr, defaults=()):
+    from typing import NamedTuple
+
+    T = NamedTuple(typename, field_descr)
+    T.__new__.__defaults__ = tuple(defaults)
+
+    # Adding the following to ensure the named tuple can be (un)pickled correctly.
+    import __main__
+
+    setattr(__main__, T.__name__, T)
+    T.__module__ = "__main__"
+    return T
+
+
+# Define the namedtuple with default here if they need to be accessible in client
+# (and w/o a real flow).
+foreach_frame_field_list = [
+    ("step", str),
+    ("var", str),
+    ("num_splits", int),
+    ("index", int),
+    ("value", str),
+]
+ForeachFrame = namedtuple_with_defaults(
+    "ForeachFrame", foreach_frame_field_list, (None,) * (len(foreach_frame_field_list))
+)

metaflow/util.py

@@ -51,21 +51,6 @@ def unquote_bytes(x):
     from shlex import quote as _quote
 
 
-from typing import NamedTuple
-
-
-def namedtuple_with_defaults(typename, field_descr, defaults=()):
-    T = NamedTuple(typename, field_descr)
-    T.__new__.__defaults__ = tuple(defaults)
-
-    # Adding the following to ensure the named tuple can be (un)pickled correctly.
-    import __main__
-
-    setattr(__main__, T.__name__, T)
-    T.__module__ = "__main__"
-    return T
-
-
 class TempDir(object):
     # Provide a temporary directory since Python 2.7 does not have it inbuilt
     def __enter__(self):