From 8e05ebc76cb6ee9b8b006354d766657cbcc3f2f0 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:04:12 -0600 Subject: [PATCH] UW-657 fs makedirs (#572) --- docs/index.rst | 15 +- docs/sections/user_guide/api/file.rst | 5 - docs/sections/user_guide/api/fs.rst | 5 + docs/sections/user_guide/api/index.rst | 2 +- docs/sections/user_guide/cli/tools/file.rst | 97 ------------- .../file/copy-exec-no-target-dir-err.cmd | 1 - .../file/copy-exec-no-target-dir-err.out | 1 - .../cli/tools/file/copy-exec-timedep.cmd | 4 - .../user_guide/cli/tools/file/copy-exec.cmd | 4 - .../user_guide/cli/tools/file/copy-help.cmd | 1 - .../user_guide/cli/tools/file/help.cmd | 1 - .../file/link-exec-no-target-dir-err.cmd | 1 - .../file/link-exec-no-target-dir-err.out | 1 - .../cli/tools/file/link-exec-timedep.cmd | 4 - .../user_guide/cli/tools/file/link-exec.cmd | 4 - .../user_guide/cli/tools/file/link-help.cmd | 1 - docs/sections/user_guide/cli/tools/fs.rst | 135 ++++++++++++++++++ .../cli/tools/{file => fs}/.gitignore | 0 .../cli/tools/{file => fs}/Makefile | 0 .../{file => fs}/copy-config-timedep.yaml | 0 .../cli/tools/{file => fs}/copy-config.yaml | 0 .../tools/fs/copy-exec-no-target-dir-err.cmd | 1 + .../tools/fs/copy-exec-no-target-dir-err.out | 3 + .../cli/tools/fs/copy-exec-timedep.cmd | 4 + .../tools/{file => fs}/copy-exec-timedep.out | 0 .../user_guide/cli/tools/fs/copy-exec.cmd | 4 + .../cli/tools/{file => fs}/copy-exec.out | 0 .../user_guide/cli/tools/fs/copy-help.cmd | 1 + .../cli/tools/{file => fs}/copy-help.out | 8 +- .../sections/user_guide/cli/tools/fs/help.cmd | 1 + .../cli/tools/{file => fs}/help.out | 6 +- .../{file => fs}/link-config-timedep.yaml | 0 .../cli/tools/{file => fs}/link-config.yaml | 0 .../tools/fs/link-exec-no-target-dir-err.cmd | 1 + .../tools/fs/link-exec-no-target-dir-err.out | 3 + .../cli/tools/fs/link-exec-timedep.cmd | 4 + .../tools/{file => fs}/link-exec-timedep.out | 0 .../user_guide/cli/tools/fs/link-exec.cmd | 4 + .../cli/tools/{file => fs}/link-exec.out | 0 .../user_guide/cli/tools/fs/link-help.cmd | 1 + .../cli/tools/{file => fs}/link-help.out | 8 +- .../cli/tools/fs/makedirs-config-timedep.yaml | 8 ++ .../cli/tools/fs/makedirs-config.yaml | 4 + .../fs/makedirs-exec-no-target-dir-err.cmd | 1 + .../fs/makedirs-exec-no-target-dir-err.out | 3 + .../cli/tools/fs/makedirs-exec-timedep.cmd | 4 + .../cli/tools/fs/makedirs-exec-timedep.out | 29 ++++ .../user_guide/cli/tools/fs/makedirs-exec.cmd | 4 + .../user_guide/cli/tools/fs/makedirs-exec.out | 21 +++ .../user_guide/cli/tools/fs/makedirs-help.cmd | 1 + .../user_guide/cli/tools/fs/makedirs-help.out | 28 ++++ .../{file => fs}/src/20240529/12/006/baz | 0 .../user_guide/cli/tools/{file => fs}/src/bar | 0 .../user_guide/cli/tools/{file => fs}/src/foo | 0 docs/sections/user_guide/cli/tools/index.rst | 2 +- docs/sections/user_guide/yaml/files.rst | 2 +- docs/sections/user_guide/yaml/index.rst | 5 +- docs/sections/user_guide/yaml/makedirs.rst | 22 +++ src/uwtools/api/{file.py => fs.py} | 47 +++++- src/uwtools/cli.py | 83 +++++++---- src/uwtools/{file.py => fs.py} | 127 ++++++++++++---- .../resources/jsonschema/makedirs.jsonschema | 16 +++ src/uwtools/strings.py | 2 + .../tests/api/{test_file.py => test_fs.py} | 17 ++- src/uwtools/tests/test_cli.py | 47 +++--- .../tests/{test_file.py => test_fs.py} | 63 +++++--- src/uwtools/tests/test_schemas.py | 17 ++- src/uwtools/tests/utils/test_tasks.py | 7 + src/uwtools/utils/tasks.py | 13 ++ 69 files changed, 638 insertions(+), 266 deletions(-) delete mode 100644 docs/sections/user_guide/api/file.rst create mode 100644 docs/sections/user_guide/api/fs.rst delete mode 100644 docs/sections/user_guide/cli/tools/file.rst delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-exec.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/copy-help.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/help.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-exec.cmd delete mode 100644 docs/sections/user_guide/cli/tools/file/link-help.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs.rst rename docs/sections/user_guide/cli/tools/{file => fs}/.gitignore (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/Makefile (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/copy-config-timedep.yaml (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/copy-config.yaml (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/copy-exec-timedep.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-exec.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/copy-exec.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/copy-help.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/copy-help.out (73%) create mode 100644 docs/sections/user_guide/cli/tools/fs/help.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/help.out (60%) rename docs/sections/user_guide/cli/tools/{file => fs}/link-config-timedep.yaml (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/link-config.yaml (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/link-exec-timedep.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/link-exec.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/link-exec.out (100%) create mode 100644 docs/sections/user_guide/cli/tools/fs/link-help.cmd rename docs/sections/user_guide/cli/tools/{file => fs}/link-help.out (73%) create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-exec.out create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd create mode 100644 docs/sections/user_guide/cli/tools/fs/makedirs-help.out rename docs/sections/user_guide/cli/tools/{file => fs}/src/20240529/12/006/baz (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/src/bar (100%) rename docs/sections/user_guide/cli/tools/{file => fs}/src/foo (100%) create mode 100644 docs/sections/user_guide/yaml/makedirs.rst rename src/uwtools/api/{file.py => fs.py} (61%) rename src/uwtools/{file.py => fs.py} (51%) create mode 100644 src/uwtools/resources/jsonschema/makedirs.jsonschema rename src/uwtools/tests/api/{test_file.py => test_fs.py} (77%) rename src/uwtools/tests/{test_file.py => test_fs.py} (69%) diff --git a/docs/index.rst b/docs/index.rst index b89e1cb05..c99bac998 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -71,22 +71,23 @@ This tool helps transform legacy configuration files templated with the atparse | :any:`CLI documentation with examples` -File Provisioning -^^^^^^^^^^^^^^^^^ +File/Directory Provisioning +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This tool helps users define the source and destination of files to be copied or linked, in the same UW YAML language used by UW drivers. +| **CLI**: ``uw fs -h`` +| **API**: ``import uwtools.api.fs`` -| :any:`CLI documentation with examples` +This tool helps users define the source and destination of files to be copied or linked, or directories to be created, in the same UW YAML language used by UW drivers. -There is a video demonstration of the use of the ``uw file`` tool available via YouTube. +| :any:`CLI documentation with examples` + +There is a video demonstration of the use of the ``uw fs`` tool (formerly ``uw file``) available via YouTube. .. raw:: html -| - Rocoto Configurability ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/sections/user_guide/api/file.rst b/docs/sections/user_guide/api/file.rst deleted file mode 100644 index d705c81bc..000000000 --- a/docs/sections/user_guide/api/file.rst +++ /dev/null @@ -1,5 +0,0 @@ -``uwtools.api.file`` -==================== - -.. automodule:: uwtools.api.file - :members: diff --git a/docs/sections/user_guide/api/fs.rst b/docs/sections/user_guide/api/fs.rst new file mode 100644 index 000000000..0ac50fc87 --- /dev/null +++ b/docs/sections/user_guide/api/fs.rst @@ -0,0 +1,5 @@ +``uwtools.api.fs`` +================== + +.. automodule:: uwtools.api.fs + :members: diff --git a/docs/sections/user_guide/api/index.rst b/docs/sections/user_guide/api/index.rst index 8caae73e8..cd29d7614 100644 --- a/docs/sections/user_guide/api/index.rst +++ b/docs/sections/user_guide/api/index.rst @@ -8,8 +8,8 @@ API driver esg_grid execute - file filter_topo + fs fv3 global_equiv_resol ioda diff --git a/docs/sections/user_guide/cli/tools/file.rst b/docs/sections/user_guide/cli/tools/file.rst deleted file mode 100644 index b7d34cbe3..000000000 --- a/docs/sections/user_guide/cli/tools/file.rst +++ /dev/null @@ -1,97 +0,0 @@ -``file`` -======== - -The ``uw`` mode for handling filesystem files. - -.. literalinclude:: file/help.cmd - :emphasize-lines: 1 -.. literalinclude:: file/help.out - :language: text - -.. _cli_file_copy_examples: - -``copy`` --------- - -The ``copy`` action stages files in a target directory by copying files. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. - -.. literalinclude:: file/copy-help.cmd - :emphasize-lines: 1 -.. literalinclude:: file/copy-help.out - :language: text - -Examples -^^^^^^^^ - -Given ``copy-config.yaml`` containing - -.. literalinclude:: file/copy-config.yaml - :language: yaml -.. literalinclude:: file/copy-exec.cmd - :emphasize-lines: 2 -.. literalinclude:: file/copy-exec.out - :language: text - -Here, ``foo`` and ``bar`` are copies of their respective source files. - -The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: - -.. literalinclude:: file/copy-config-timedep.yaml - :language: yaml -.. literalinclude:: file/copy-exec-timedep.cmd - :emphasize-lines: 2 -.. literalinclude:: file/copy-exec-timedep.out - :language: text - -The ``--target-dir`` option is optional when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: - -.. literalinclude:: file/copy-config.yaml - :language: yaml -.. literalinclude:: file/copy-exec-no-target-dir-err.cmd - :emphasize-lines: 1 -.. literalinclude:: file/copy-exec-no-target-dir-err.out - :language: text - -.. _cli_file_link_examples: - -``link`` --------- - -The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. - -.. literalinclude:: file/link-help.cmd - :emphasize-lines: 1 -.. literalinclude:: file/link-help.out - :language: text - -Examples -^^^^^^^^ - -Given ``link-config.yaml`` containing - -.. literalinclude:: file/link-config.yaml - :language: yaml -.. literalinclude:: file/link-exec.cmd - :emphasize-lines: 2 -.. literalinclude:: file/link-exec.out - :language: text - -Here, ``foo`` and ``bar`` are symbolic links. - -The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: - -.. literalinclude:: file/link-config-timedep.yaml - :language: yaml -.. literalinclude:: file/link-exec-timedep.cmd - :emphasize-lines: 2 -.. literalinclude:: file/link-exec-timedep.out - :language: text - -The ``--target-dir`` option is optional when all linkname paths are absolute, and will never be applied to absolute linkname paths. If any linkname paths are relative, however, it is an error not to provide a target directory: - -.. literalinclude:: file/link-config.yaml - :language: yaml -.. literalinclude:: file/link-exec-no-target-dir-err.cmd - :emphasize-lines: 1 -.. literalinclude:: file/link-exec-no-target-dir-err.out - :language: text diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd deleted file mode 100644 index ed4fd1f90..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file copy --config-file copy-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out deleted file mode 100644 index 388dc0318..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec-no-target-dir-err.out +++ /dev/null @@ -1 +0,0 @@ -[2024-07-26T21:51:23] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd deleted file mode 100644 index cec34d0a7..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf copy-dst-timedep -uw file copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files -echo -tree copy-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec.cmd b/docs/sections/user_guide/cli/tools/file/copy-exec.cmd deleted file mode 100644 index f74e38e33..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf copy-dst -uw file copy --target-dir copy-dst --config-file copy-config.yaml config files -echo -tree copy-dst diff --git a/docs/sections/user_guide/cli/tools/file/copy-help.cmd b/docs/sections/user_guide/cli/tools/file/copy-help.cmd deleted file mode 100644 index 5a5188caa..000000000 --- a/docs/sections/user_guide/cli/tools/file/copy-help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file copy --help diff --git a/docs/sections/user_guide/cli/tools/file/help.cmd b/docs/sections/user_guide/cli/tools/file/help.cmd deleted file mode 100644 index e89f12e45..000000000 --- a/docs/sections/user_guide/cli/tools/file/help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file --help diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd deleted file mode 100644 index 83b52f5e5..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file link --config-file link-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out deleted file mode 100644 index 388dc0318..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec-no-target-dir-err.out +++ /dev/null @@ -1 +0,0 @@ -[2024-07-26T21:51:23] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd deleted file mode 100644 index 915899c83..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst-timedep -uw file link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files -echo -tree link-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/link-exec.cmd b/docs/sections/user_guide/cli/tools/file/link-exec.cmd deleted file mode 100644 index 81da785b4..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst -uw file link --target-dir link-dst --config-file link-config.yaml config files -echo -tree link-dst diff --git a/docs/sections/user_guide/cli/tools/file/link-help.cmd b/docs/sections/user_guide/cli/tools/file/link-help.cmd deleted file mode 100644 index a3ce9a824..000000000 --- a/docs/sections/user_guide/cli/tools/file/link-help.cmd +++ /dev/null @@ -1 +0,0 @@ -uw file link --help diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst new file mode 100644 index 000000000..cb61dd7bf --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -0,0 +1,135 @@ +``fs`` +====== + +.. _cli_fs_mode: + +The ``uw`` mode for handling filesystem items (files and directories). + +.. literalinclude:: fs/help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/help.out + :language: text + +``copy`` +-------- + +The ``copy`` action stages files in a target directory by copying files. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. + +.. literalinclude:: fs/copy-help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/copy-help.out + :language: text + +Examples +^^^^^^^^ + +Given ``copy-config.yaml`` containing + +.. literalinclude:: fs/copy-config.yaml + :language: yaml +.. literalinclude:: fs/copy-exec.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/copy-exec.out + :language: text + +Here, ``foo`` and ``bar`` are copies of their respective source files. + +The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: + +.. literalinclude:: fs/copy-config-timedep.yaml + :language: yaml +.. literalinclude:: fs/copy-exec-timedep.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/copy-exec-timedep.out + :language: text + +The ``--target-dir`` option need not be specified when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: + +.. literalinclude:: fs/copy-config.yaml + :language: yaml +.. literalinclude:: fs/copy-exec-no-target-dir-err.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/copy-exec-no-target-dir-err.out + :language: text + +``link`` +-------- + +The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. + +.. literalinclude:: fs/link-help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/link-help.out + :language: text + +Examples +^^^^^^^^ + +Given ``link-config.yaml`` containing + +.. literalinclude:: fs/link-config.yaml + :language: yaml +.. literalinclude:: fs/link-exec.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/link-exec.out + :language: text + +Here, ``foo`` and ``bar`` are symbolic links. + +The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: + +.. literalinclude:: fs/link-config-timedep.yaml + :language: yaml +.. literalinclude:: fs/link-exec-timedep.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/link-exec-timedep.out + :language: text + +The ``--target-dir`` option need not be specified when all linkname paths are absolute, and will never be applied to absolute linkname paths. If any linkname paths are relative, however, it is an error not to provide a target directory: + +.. literalinclude:: fs/link-config.yaml + :language: yaml +.. literalinclude:: fs/link-exec-no-target-dir-err.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/link-exec-no-target-dir-err.out + :language: text + +``makedirs`` +------------ + +The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block `. + +.. literalinclude:: fs/makedirs-help.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/makedirs-help.out + :language: text + +Examples +^^^^^^^^ + +Given ``makedirs-config.yaml`` containing + +.. literalinclude:: fs/makedirs-config.yaml + :language: yaml +.. literalinclude:: fs/makedirs-exec.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/makedirs-exec.out + :language: text + +The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: + +.. literalinclude:: fs/makedirs-config-timedep.yaml + :language: yaml +.. literalinclude:: fs/makedirs-exec-timedep.cmd + :emphasize-lines: 2 +.. literalinclude:: fs/makedirs-exec-timedep.out + :language: text + +The ``--target-dir`` option need not be specified when all directory paths are absolute, and will never be applied to absolute paths. If any paths are relative, however, it is an error not to provide a target directory: + +.. literalinclude:: fs/makedirs-config.yaml + :language: yaml +.. literalinclude:: fs/makedirs-exec-no-target-dir-err.cmd + :emphasize-lines: 1 +.. literalinclude:: fs/makedirs-exec-no-target-dir-err.out + :language: text diff --git a/docs/sections/user_guide/cli/tools/file/.gitignore b/docs/sections/user_guide/cli/tools/fs/.gitignore similarity index 100% rename from docs/sections/user_guide/cli/tools/file/.gitignore rename to docs/sections/user_guide/cli/tools/fs/.gitignore diff --git a/docs/sections/user_guide/cli/tools/file/Makefile b/docs/sections/user_guide/cli/tools/fs/Makefile similarity index 100% rename from docs/sections/user_guide/cli/tools/file/Makefile rename to docs/sections/user_guide/cli/tools/fs/Makefile diff --git a/docs/sections/user_guide/cli/tools/file/copy-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml diff --git a/docs/sections/user_guide/cli/tools/file/copy-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd new file mode 100644 index 000000000..fb3d8b83f --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs copy --config-file copy-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out new file mode 100644 index 000000000..02c57878d --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out @@ -0,0 +1,3 @@ +[2024-08-14T15:19:59] INFO Validating config against internal schema: files-to-stage +[2024-08-14T15:19:59] INFO 0 UW schema-validation errors found +[2024-08-14T15:19:59] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd new file mode 100644 index 000000000..0f2a1911b --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd @@ -0,0 +1,4 @@ +rm -rf copy-dst-timedep +uw fs copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files +echo +tree copy-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-exec-timedep.out rename to docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd new file mode 100644 index 000000000..175451030 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd @@ -0,0 +1,4 @@ +rm -rf copy-dst +uw fs copy --target-dir copy-dst --config-file copy-config.yaml config files +echo +tree copy-dst diff --git a/docs/sections/user_guide/cli/tools/file/copy-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/copy-exec.out rename to docs/sections/user_guide/cli/tools/fs/copy-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-help.cmd b/docs/sections/user_guide/cli/tools/fs/copy-help.cmd new file mode 100644 index 000000000..4806f822a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-help.cmd @@ -0,0 +1 @@ +uw fs copy --help diff --git a/docs/sections/user_guide/cli/tools/file/copy-help.out b/docs/sections/user_guide/cli/tools/fs/copy-help.out similarity index 73% rename from docs/sections/user_guide/cli/tools/file/copy-help.out rename to docs/sections/user_guide/cli/tools/fs/copy-help.out index bbc885512..ce73007a6 100644 --- a/docs/sections/user_guide/cli/tools/file/copy-help.out +++ b/docs/sections/user_guide/cli/tools/fs/copy-help.out @@ -1,7 +1,7 @@ -usage: uw file copy [-h] [--version] [--config-file PATH] [--target-dir PATH] - [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] - [--quiet] [--verbose] - [KEY ...] +usage: uw fs copy [-h] [--version] [--config-file PATH] [--target-dir PATH] + [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] [--quiet] + [--verbose] + [KEY ...] Copy files diff --git a/docs/sections/user_guide/cli/tools/fs/help.cmd b/docs/sections/user_guide/cli/tools/fs/help.cmd new file mode 100644 index 000000000..7e46d94b9 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/help.cmd @@ -0,0 +1 @@ +uw fs --help diff --git a/docs/sections/user_guide/cli/tools/file/help.out b/docs/sections/user_guide/cli/tools/fs/help.out similarity index 60% rename from docs/sections/user_guide/cli/tools/file/help.out rename to docs/sections/user_guide/cli/tools/fs/help.out index 70252750d..6d573c404 100644 --- a/docs/sections/user_guide/cli/tools/file/help.out +++ b/docs/sections/user_guide/cli/tools/fs/help.out @@ -1,6 +1,6 @@ -usage: uw file [-h] [--version] ACTION ... +usage: uw fs [-h] [--version] ACTION ... -Handle files +Handle filesystem items (files and directories) Optional arguments: -h, --help @@ -14,3 +14,5 @@ Positional arguments: Copy files link Link files + makedirs + Make directories diff --git a/docs/sections/user_guide/cli/tools/file/link-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml diff --git a/docs/sections/user_guide/cli/tools/file/link-config.yaml b/docs/sections/user_guide/cli/tools/fs/link-config.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-config.yaml rename to docs/sections/user_guide/cli/tools/fs/link-config.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd new file mode 100644 index 000000000..8446a4e8b --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs link --config-file link-config.yaml config files diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out new file mode 100644 index 000000000..03c1247f5 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out @@ -0,0 +1,3 @@ +[2024-08-14T15:19:57] INFO Validating config against internal schema: files-to-stage +[2024-08-14T15:19:57] INFO 0 UW schema-validation errors found +[2024-08-14T15:19:57] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd new file mode 100644 index 000000000..f3e240397 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd @@ -0,0 +1,4 @@ +rm -rf link-dst-timedep +uw fs link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config files +echo +tree link-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/file/link-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-exec-timedep.out rename to docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd new file mode 100644 index 000000000..f4f14059b --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd @@ -0,0 +1,4 @@ +rm -rf link-dst +uw fs link --target-dir link-dst --config-file link-config.yaml config files +echo +tree link-dst diff --git a/docs/sections/user_guide/cli/tools/file/link-exec.out b/docs/sections/user_guide/cli/tools/fs/link-exec.out similarity index 100% rename from docs/sections/user_guide/cli/tools/file/link-exec.out rename to docs/sections/user_guide/cli/tools/fs/link-exec.out diff --git a/docs/sections/user_guide/cli/tools/fs/link-help.cmd b/docs/sections/user_guide/cli/tools/fs/link-help.cmd new file mode 100644 index 000000000..b36d8d6d2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/link-help.cmd @@ -0,0 +1 @@ +uw fs link --help diff --git a/docs/sections/user_guide/cli/tools/file/link-help.out b/docs/sections/user_guide/cli/tools/fs/link-help.out similarity index 73% rename from docs/sections/user_guide/cli/tools/file/link-help.out rename to docs/sections/user_guide/cli/tools/fs/link-help.out index 8734b5ad4..be07a9eec 100644 --- a/docs/sections/user_guide/cli/tools/file/link-help.out +++ b/docs/sections/user_guide/cli/tools/fs/link-help.out @@ -1,7 +1,7 @@ -usage: uw file link [-h] [--version] [--config-file PATH] [--target-dir PATH] - [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] - [--quiet] [--verbose] - [KEY ...] +usage: uw fs link [-h] [--version] [--config-file PATH] [--target-dir PATH] + [--cycle CYCLE] [--leadtime LEADTIME] [--dry-run] [--quiet] + [--verbose] + [KEY ...] Link files diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml new file mode 100644 index 000000000..abf1001b8 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml @@ -0,0 +1,8 @@ +config: + makedirs: + - foo/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/bar + - baz/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/qux +yyyymmdd: "{{ cycle.strftime('%Y%m%d') }}" +hh: "{{ cycle.strftime('%H') }}" +nnn: "{{ '%03d' % (leadtime.total_seconds() // 3600) }}" +validtime: "{{ (cycle + leadtime).strftime('%Y-%m-%dT%H') }}" diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml new file mode 100644 index 000000000..6bf12f29a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml @@ -0,0 +1,4 @@ +config: + makedirs: + - foo + - bar diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd new file mode 100644 index 000000000..80ff4150e --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs makedirs --config-file makedirs-config.yaml config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out new file mode 100644 index 000000000..ceefc693e --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out @@ -0,0 +1,3 @@ +[2024-08-14T15:19:58] INFO Validating config against internal schema: makedirs +[2024-08-14T15:19:58] INFO 0 UW schema-validation errors found +[2024-08-14T15:19:58] ERROR Relative path 'foo' requires the target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd new file mode 100644 index 000000000..fa30614ec --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd @@ -0,0 +1,4 @@ +rm -rf makedirs-parent-timedep +uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 config +echo +tree -F makedirs-parent-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out new file mode 100644 index 000000000..a89e33a29 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out @@ -0,0 +1,29 @@ +[2024-08-12T04:35:49] INFO Validating config against internal schema: makedirs +[2024-08-12T04:35:49] INFO 0 UW schema-validation errors found +[2024-08-12T04:35:49] INFO Directories: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directories: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/foo/20240529/12/006/bar: Final state: Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent-timedep/baz/20240529/12/006/qux: Final state: Ready +[2024-08-12T04:35:49] INFO Directories: Final state: Ready + +makedirs-parent-timedep/ +├── baz/ +│   └── 20240529/ +│   └── 12/ +│   └── 006/ +│   └── qux/ +└── foo/ + └── 20240529/ + └── 12/ + └── 006/ + └── bar/ + +11 directories, 0 files diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd new file mode 100644 index 000000000..b99d8c3f4 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd @@ -0,0 +1,4 @@ +rm -rf makedirs-parent +uw fs makedirs --target-dir makedirs-parent --config-file makedirs-config.yaml config +echo +tree -F makedirs-parent diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out new file mode 100644 index 000000000..376518b59 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out @@ -0,0 +1,21 @@ +[2024-08-12T04:35:49] INFO Validating config against internal schema: makedirs +[2024-08-12T04:35:49] INFO 0 UW schema-validation errors found +[2024-08-12T04:35:49] INFO Directories: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directories: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent/foo: Final state: Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Initial state: Not Ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Checking requirements +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Requirement(s) ready +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Executing +[2024-08-12T04:35:49] INFO Directory makedirs-parent/bar: Final state: Ready +[2024-08-12T04:35:49] INFO Directories: Final state: Ready + +makedirs-parent/ +├── bar/ +└── foo/ + +3 directories, 0 files diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd new file mode 100644 index 000000000..f9cacc5e3 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-help.cmd @@ -0,0 +1 @@ +uw fs makedirs --help diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-help.out b/docs/sections/user_guide/cli/tools/fs/makedirs-help.out new file mode 100644 index 000000000..0e80d2f16 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-help.out @@ -0,0 +1,28 @@ +usage: uw fs makedirs [-h] [--version] [--config-file PATH] + [--target-dir PATH] [--cycle CYCLE] + [--leadtime LEADTIME] [--dry-run] [--quiet] [--verbose] + [KEY ...] + +Make directories + +Optional arguments: + -h, --help + Show help and exit + --version + Show version info and exit + --config-file PATH, -c PATH + Path to UW YAML config file (default: read from stdin) + --target-dir PATH + Root directory for relative destination paths + --cycle CYCLE + The cycle in ISO8601 format (e.g. 2024-08-12T00) + --leadtime LEADTIME + The leadtime as hours[:minutes[:seconds]] + --dry-run + Only log info, making no changes + --quiet, -q + Print no logging messages + --verbose, -v + Print all logging messages + KEY + YAML key leading to file dst/src block diff --git a/docs/sections/user_guide/cli/tools/file/src/20240529/12/006/baz b/docs/sections/user_guide/cli/tools/fs/src/20240529/12/006/baz similarity index 100% rename from docs/sections/user_guide/cli/tools/file/src/20240529/12/006/baz rename to docs/sections/user_guide/cli/tools/fs/src/20240529/12/006/baz diff --git a/docs/sections/user_guide/cli/tools/file/src/bar b/docs/sections/user_guide/cli/tools/fs/src/bar similarity index 100% rename from docs/sections/user_guide/cli/tools/file/src/bar rename to docs/sections/user_guide/cli/tools/fs/src/bar diff --git a/docs/sections/user_guide/cli/tools/file/src/foo b/docs/sections/user_guide/cli/tools/fs/src/foo similarity index 100% rename from docs/sections/user_guide/cli/tools/file/src/foo rename to docs/sections/user_guide/cli/tools/fs/src/foo diff --git a/docs/sections/user_guide/cli/tools/index.rst b/docs/sections/user_guide/cli/tools/index.rst index 27d52c00f..238759a8b 100644 --- a/docs/sections/user_guide/cli/tools/index.rst +++ b/docs/sections/user_guide/cli/tools/index.rst @@ -6,6 +6,6 @@ Tools config execute - file + fs rocoto template diff --git a/docs/sections/user_guide/yaml/files.rst b/docs/sections/user_guide/yaml/files.rst index 878ffd382..0d34ba38d 100644 --- a/docs/sections/user_guide/yaml/files.rst +++ b/docs/sections/user_guide/yaml/files.rst @@ -3,7 +3,7 @@ File Blocks =========== -File blocks define files to be staged in a target directory as copies or symbolic links. Keys in such blocks specify destination paths relative to the target directory, and values specify source paths. +File blocks define files to be staged in a target directory as copies or symbolic links. Keys in such blocks specify either absolute destination paths, or destination paths relative to the target directory. Values specify source paths. Example block: diff --git a/docs/sections/user_guide/yaml/index.rst b/docs/sections/user_guide/yaml/index.rst index 8da086673..e8e1be934 100644 --- a/docs/sections/user_guide/yaml/index.rst +++ b/docs/sections/user_guide/yaml/index.rst @@ -5,9 +5,10 @@ UW YAML :maxdepth: 1 components/index - platform execution files - updating_values + makedirs + platform rocoto tags + updating_values diff --git a/docs/sections/user_guide/yaml/makedirs.rst b/docs/sections/user_guide/yaml/makedirs.rst new file mode 100644 index 000000000..e1744666a --- /dev/null +++ b/docs/sections/user_guide/yaml/makedirs.rst @@ -0,0 +1,22 @@ +.. _makedirs_yaml: + +Directory Blocks +================ + +Directory blocks define a sequence of one or more directories to be created, nested under a ``makedirs:`` key. Each value is either an absolute path, or a path relative to the target directory, specified either via the CLI or an API call. + +Example block with absolute paths: + +.. code-block:: yaml + + makedirs: + - /path/to/dir1 + - /path/to/dir2 + +Example block with relative paths: + +.. code-block:: yaml + + makedirs: + - /subdir/dir1 + - ../../dir2 diff --git a/src/uwtools/api/file.py b/src/uwtools/api/fs.py similarity index 61% rename from src/uwtools/api/file.py rename to src/uwtools/api/fs.py index 7bac2ddef..5b7d6102f 100644 --- a/src/uwtools/api/file.py +++ b/src/uwtools/api/fs.py @@ -1,5 +1,5 @@ """ -API access to ``uwtools`` file-management tools. +API access to ``uwtools`` file and directory management tools. """ import datetime as dt @@ -8,7 +8,7 @@ from iotaa import Asset -from uwtools.file import Copier, Linker +from uwtools.fs import Copier, Linker, MakeDirs from uwtools.utils.api import ensure_data_source as _ensure_data_source @@ -33,7 +33,7 @@ def copy( :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if all copies were created. """ - copier = Copier( + stager = Copier( target_dir=Path(target_dir) if target_dir else None, config=_ensure_data_source(config, stdin_ok), cycle=cycle, @@ -41,7 +41,7 @@ def copy( keys=keys, dry_run=dry_run, ) - assets: list[Asset] = copier.go() # type: ignore + assets: list[Asset] = stager.go() # type: ignore return all(asset.ready() for asset in assets) @@ -66,7 +66,7 @@ def link( :param stdin_ok: OK to read from ``stdin``? :return: ``True`` if all links were created. """ - linker = Linker( + stager = Linker( target_dir=Path(target_dir) if target_dir else None, config=_ensure_data_source(config, stdin_ok), cycle=cycle, @@ -74,8 +74,41 @@ def link( keys=keys, dry_run=dry_run, ) - assets: list[Asset] = linker.go() # type: ignore + assets: list[Asset] = stager.go() # type: ignore return all(asset.ready() for asset in assets) -__all__ = ["Copier", "Linker", "copy", "link"] +def makedirs( + config: Optional[Union[Path, dict, str]] = None, + target_dir: Optional[Union[Path, str]] = None, + cycle: Optional[dt.datetime] = None, + leadtime: Optional[dt.timedelta] = None, + keys: Optional[list[str]] = None, + dry_run: bool = False, + stdin_ok: bool = False, +) -> bool: + """ + Make directories. + + :param config: YAML-file path, or ``dict`` (read ``stdin`` if missing or ``None``). + :param target_dir: Path to target directory. + :param cycle: A datetime object to make available for use in the config. + :param leadtime: A timedelta object to make available for use in the config. + :param keys: YAML keys leading to file dst/src block. + :param dry_run: Do not link files. + :param stdin_ok: OK to read from ``stdin``? + :return: ``True`` if all directories were made. + """ + stager = MakeDirs( + target_dir=Path(target_dir) if target_dir else None, + config=_ensure_data_source(config, stdin_ok), + cycle=cycle, + leadtime=leadtime, + keys=keys, + dry_run=dry_run, + ) + assets: list[Asset] = stager.go() # type: ignore + return all(asset.ready() for asset in assets) + + +__all__ = ["Copier", "Linker", "MakeDirs", "copy", "link", "makedirs"] diff --git a/src/uwtools/cli.py b/src/uwtools/cli.py index 3d47da43e..7edce9f02 100644 --- a/src/uwtools/cli.py +++ b/src/uwtools/cli.py @@ -19,7 +19,7 @@ import uwtools.api.config import uwtools.api.driver import uwtools.api.execute -import uwtools.api.file +import uwtools.api.fs import uwtools.api.rocoto import uwtools.api.template import uwtools.config.jinja2 @@ -62,7 +62,7 @@ def main() -> None: tools: dict[str, Callable[..., bool]] = { STR.config: _dispatch_config, STR.execute: _dispatch_execute, - STR.file: _dispatch_file, + STR.fs: _dispatch_fs, STR.rocoto: _dispatch_rocoto, STR.template: _dispatch_template, } @@ -305,27 +305,28 @@ def _dispatch_execute(args: Args) -> bool: ) -# Mode file +# Mode fs -def _add_subparser_file(subparsers: Subparsers) -> ModeChecks: +def _add_subparser_fs(subparsers: Subparsers) -> ModeChecks: """ - Subparser for mode: file + Subparser for mode: fs :param subparsers: Parent parser's subparsers, to add this subparser to. """ - parser = _add_subparser(subparsers, STR.file, "Handle files") + parser = _add_subparser(subparsers, STR.fs, "Handle filesystem items (files and directories)") _basic_setup(parser) subparsers = _add_subparsers(parser, STR.action, STR.action.upper()) return { - STR.copy: _add_subparser_file_copy(subparsers), - STR.link: _add_subparser_file_link(subparsers), + STR.copy: _add_subparser_fs_copy(subparsers), + STR.link: _add_subparser_fs_link(subparsers), + STR.makedirs: _add_subparser_fs_makedirs(subparsers), } -def _add_subparser_file_common(parser: Parser) -> ActionChecks: +def _add_subparser_fs_common(parser: Parser) -> ActionChecks: """ - Common subparser code for mode: file {copy link} + Common subparser code for mode: fs {copy link makedirs} :param parser: The parser to configure. """ @@ -340,46 +341,74 @@ def _add_subparser_file_common(parser: Parser) -> ActionChecks: return checks -def _add_subparser_file_copy(subparsers: Subparsers) -> ActionChecks: +def _add_subparser_fs_copy(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: file copy + Subparser for mode: fs copy :param subparsers: Parent parser's subparsers, to add this subparser to. """ parser = _add_subparser(subparsers, STR.copy, "Copy files") - return _add_subparser_file_common(parser) + return _add_subparser_fs_common(parser) -def _add_subparser_file_link(subparsers: Subparsers) -> ActionChecks: +def _add_subparser_fs_link(subparsers: Subparsers) -> ActionChecks: """ - Subparser for mode: file link + Subparser for mode: fs link :param subparsers: Parent parser's subparsers, to add this subparser to. """ parser = _add_subparser(subparsers, STR.link, "Link files") - return _add_subparser_file_common(parser) + return _add_subparser_fs_common(parser) -def _dispatch_file(args: Args) -> bool: +def _add_subparser_fs_makedirs(subparsers: Subparsers) -> ActionChecks: """ - Dispatch logic for file mode. + Subparser for mode: fs makedirs + + :param subparsers: Parent parser's subparsers, to add this subparser to. + """ + parser = _add_subparser(subparsers, STR.makedirs, "Make directories") + return _add_subparser_fs_common(parser) + + +def _dispatch_fs(args: Args) -> bool: + """ + Dispatch logic for fs mode. :param args: Parsed command-line args. """ actions = { - STR.copy: _dispatch_file_copy, - STR.link: _dispatch_file_link, + STR.copy: _dispatch_fs_copy, + STR.link: _dispatch_fs_link, + STR.makedirs: _dispatch_fs_makedirs, } return actions[args[STR.action]](args) -def _dispatch_file_copy(args: Args) -> bool: +def _dispatch_fs_copy(args: Args) -> bool: + """ + Dispatch logic for fs copy action. + + :param args: Parsed command-line args. + """ + return uwtools.api.fs.copy( + target_dir=args[STR.targetdir], + config=args[STR.cfgfile], + cycle=args[STR.cycle], + leadtime=args[STR.leadtime], + keys=args[STR.keys], + dry_run=args[STR.dryrun], + stdin_ok=True, + ) + + +def _dispatch_fs_link(args: Args) -> bool: """ - Dispatch logic for file copy action. + Dispatch logic for fs link action. :param args: Parsed command-line args. """ - return uwtools.api.file.copy( + return uwtools.api.fs.link( target_dir=args[STR.targetdir], config=args[STR.cfgfile], cycle=args[STR.cycle], @@ -390,13 +419,13 @@ def _dispatch_file_copy(args: Args) -> bool: ) -def _dispatch_file_link(args: Args) -> bool: +def _dispatch_fs_makedirs(args: Args) -> bool: """ - Dispatch logic for file link action. + Dispatch logic for fs makedirs action. :param args: Parsed command-line args. """ - return uwtools.api.file.link( + return uwtools.api.fs.makedirs( target_dir=args[STR.targetdir], config=args[STR.cfgfile], cycle=args[STR.cycle], @@ -1115,7 +1144,7 @@ def _parse_args(raw_args: list[str]) -> tuple[Args, Checks]: tools = { STR.config: partial(_add_subparser_config, subparsers), STR.execute: partial(_add_subparser_execute, subparsers), - STR.file: partial(_add_subparser_file, subparsers), + STR.fs: partial(_add_subparser_fs, subparsers), STR.rocoto: partial(_add_subparser_rocoto, subparsers), STR.template: partial(_add_subparser_template, subparsers), } diff --git a/src/uwtools/file.py b/src/uwtools/fs.py similarity index 51% rename from src/uwtools/file.py rename to src/uwtools/fs.py index 5b8162d8b..9ef5de151 100644 --- a/src/uwtools/file.py +++ b/src/uwtools/fs.py @@ -1,9 +1,9 @@ """ -File handling. +File and directory staging. """ import datetime as dt -from functools import cached_property +from abc import ABC, abstractmethod from pathlib import Path from typing import Optional, Union @@ -13,13 +13,14 @@ from uwtools.config.validator import validate_internal from uwtools.exceptions import UWConfigError from uwtools.logging import log +from uwtools.strings import STR from uwtools.utils.api import str2path -from uwtools.utils.tasks import filecopy, symlink +from uwtools.utils.tasks import directory, filecopy, symlink -class Stager: +class Stager(ABC): """ - The base class for staging files. + The base class for staging files and directories. """ def __init__( @@ -32,7 +33,7 @@ def __init__( dry_run: bool = False, ) -> None: """ - Handle files. + Stage files and directories. :param config: YAML-file path, or dict (read stdin if missing or None). :param target_dir: Path to target directory. @@ -43,39 +44,41 @@ def __init__( :raises: UWConfigError if config fails validation. """ dryrun(enable=dry_run) - self._target_dir = str2path(target_dir) - self._config = YAMLConfig(config=str2path(config)) self._keys = keys or [] - self._config.dereference( + self._target_dir = str2path(target_dir) + yaml_config = YAMLConfig(config=str2path(config)) + yaml_config.dereference( context={ **({"cycle": cycle} if cycle else {}), **({"leadtime": leadtime} if leadtime else {}), - **self._config.data, + **yaml_config.data, } ) + self._config = yaml_config.data + self._set_config_block() self._validate() + self._check_paths() - def _check_dst_paths(self, cfg: dict[str, str]) -> None: + def _check_paths(self) -> None: """ - Check that all destination paths are absolute if no target directory is specified. + Check that all paths are absolute if no target directory is specified. - :parm cfg: The dst/linkname -> src/target map. + :parm paths: The paths to check. :raises: UWConfigError if no target directory is specified and a relative path is. """ if not self._target_dir: errmsg = "Relative path '%s' requires the target directory to be specified" - for dst in cfg.keys(): + for dst in self._dst_paths: if not Path(dst).is_absolute(): raise UWConfigError(errmsg % dst) - @cached_property - def _file_map(self) -> dict: + def _set_config_block(self) -> None: """ - Navigate keys to file dst/src config block. + Navigate keys to a config block. - :return: The dst/src file block from a potentially larger config. + :raises: UWConfigError if no target directory is specified and a relative path is. """ - cfg = self._config.data + cfg = self._config nav = [] for key in self._keys: nav.append(key) @@ -84,22 +87,54 @@ def _file_map(self) -> dict: log.debug("Following config key '%s'", key) cfg = cfg[key] if not isinstance(cfg, dict): - raise UWConfigError("No file map found at key path: %s" % " -> ".join(self._keys)) - self._check_dst_paths(cfg) - return cfg + msg = "Expected block not found at key path: %s" % " -> ".join(self._keys) + raise UWConfigError(msg) + self._config = cfg + + @property + @abstractmethod + def _dst_paths(self) -> list[str]: + """ + Returns the paths to files or directories to create. + """ + + @property + @abstractmethod + def _schema(self) -> str: + """ + Returns the name of the schema to use for config validation. + """ - def _validate(self) -> bool: + def _validate(self) -> None: """ Validate config against its schema. - :return: True if config passes validation. :raises: UWConfigError if config fails validation. """ - validate_internal(schema_name="files-to-stage", config=self._file_map) - return True + validate_internal(schema_name=self._schema, config=self._config) -class Copier(Stager): +class FileStager(Stager): + """ + Stage files. + """ + + @property + def _dst_paths(self) -> list[str]: + """ + Returns the paths to files to create. + """ + return list(self._config.keys()) + + @property + def _schema(self) -> str: + """ + Returns the name of the schema to use for config validation. + """ + return "files-to-stage" + + +class Copier(FileStager): """ Stage files by copying. """ @@ -111,10 +146,10 @@ def go(self): """ dst = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File copies" - yield [filecopy(src=Path(v), dst=dst(k)) for k, v in self._file_map.items()] + yield [filecopy(src=Path(v), dst=dst(k)) for k, v in self._config.items()] -class Linker(Stager): +class Linker(FileStager): """ Stage files by linking. """ @@ -126,4 +161,36 @@ def go(self): """ linkname = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File links" - yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._file_map.items()] + yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._config.items()] + + +class MakeDirs(Stager): + """ + Make directories. + """ + + @tasks + def go(self): + """ + Make directories. + """ + yield "Directories" + yield [ + directory(path=Path(self._target_dir / p if self._target_dir else p)) + for p in self._config[STR.makedirs] + ] + + @property + def _dst_paths(self) -> list[str]: + """ + Returns the paths to directories to create. + """ + paths: list[str] = self._config[STR.makedirs] + return paths + + @property + def _schema(self) -> str: + """ + Returns the name of the schema to use for config validation. + """ + return "makedirs" diff --git a/src/uwtools/resources/jsonschema/makedirs.jsonschema b/src/uwtools/resources/jsonschema/makedirs.jsonschema new file mode 100644 index 000000000..0fee1c309 --- /dev/null +++ b/src/uwtools/resources/jsonschema/makedirs.jsonschema @@ -0,0 +1,16 @@ +{ + "additionalProperties": false, + "properties": { + "makedirs": { + "items": { + "type": "string" + }, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "makedirs" + ], + "type": "object" +} diff --git a/src/uwtools/strings.py b/src/uwtools/strings.py index 3319eb592..5d3d8cd49 100644 --- a/src/uwtools/strings.py +++ b/src/uwtools/strings.py @@ -86,6 +86,7 @@ class STR: file2path: str = "file_2_path" file: str = "file" filtertopo: str = "filter_topo" + fs: str = "fs" fv3: str = "fv3" globalequivresol: str = "global_equiv_resol" graphfile: str = "graph_file" @@ -99,6 +100,7 @@ class STR: keyvalpairs: str = "key_eq_val_pairs" leadtime: str = "leadtime" link: str = "link" + makedirs: str = "makedirs" makehgrid: str = "make_hgrid" makesolomosaic: str = "make_solo_mosaic" mode: str = "mode" diff --git a/src/uwtools/tests/api/test_file.py b/src/uwtools/tests/api/test_fs.py similarity index 77% rename from src/uwtools/tests/api/test_file.py rename to src/uwtools/tests/api/test_fs.py index a403bb99b..71d48b5cb 100644 --- a/src/uwtools/tests/api/test_file.py +++ b/src/uwtools/tests/api/test_fs.py @@ -5,7 +5,7 @@ from pytest import fixture -from uwtools.api import file +from uwtools.api import fs @fixture @@ -31,7 +31,7 @@ def test_copy_fail(kwargs): for p in paths: assert not Path(p).exists() Path(list(paths.values())[0]).unlink() - assert file.copy(**kwargs) is False + assert fs.copy(**kwargs) is False assert not Path(list(paths.keys())[0]).exists() assert Path(list(paths.keys())[1]).is_file() @@ -40,7 +40,7 @@ def test_copy_pass(kwargs): paths = kwargs["config"]["a"]["b"] for p in paths: assert not Path(p).exists() - assert file.copy(**kwargs) is True + assert fs.copy(**kwargs) is True for p in paths: assert Path(p).is_file() @@ -50,7 +50,7 @@ def test_link_fail(kwargs): for p in paths: assert not Path(p).exists() Path(list(paths.values())[0]).unlink() - assert file.link(**kwargs) is False + assert fs.link(**kwargs) is False assert not Path(list(paths.keys())[0]).exists() assert Path(list(paths.keys())[1]).is_symlink() @@ -59,6 +59,13 @@ def test_link_pass(kwargs): paths = kwargs["config"]["a"]["b"] for p in paths: assert not Path(p).exists() - assert file.link(**kwargs) is True + assert fs.link(**kwargs) is True for p in paths: assert Path(p).is_symlink() + + +def test_makedirs(tmp_path): + paths = [tmp_path / "foo" / x for x in ("bar", "baz")] + assert not any(path.is_dir() for path in paths) + assert fs.makedirs(config={"makedirs": [str(path) for path in paths]}) is True + assert all(path.is_dir() for path in paths) diff --git a/src/uwtools/tests/test_cli.py b/src/uwtools/tests/test_cli.py index c8f5f08f6..d1c83000f 100644 --- a/src/uwtools/tests/test_cli.py +++ b/src/uwtools/tests/test_cli.py @@ -52,7 +52,7 @@ def args_config_realize(): @fixture -def args_dispatch_file(): +def args_dispatch_fs(): return { "target_dir": "/target/dir", "config_file": "/config/file", @@ -102,17 +102,17 @@ def test__add_subparser_config_validate(subparsers): def test__add_subparser_file(subparsers): - cli._add_subparser_file(subparsers) - assert actions(subparsers.choices[STR.file]) == [STR.copy, STR.link] + cli._add_subparser_fs(subparsers) + assert actions(subparsers.choices[STR.fs]) == [STR.copy, STR.link, STR.makedirs] def test__add_subparser_file_copy(subparsers): - cli._add_subparser_file_copy(subparsers) + cli._add_subparser_fs_copy(subparsers) assert subparsers.choices[STR.copy] def test__add_subparser_file_link(subparsers): - cli._add_subparser_file_link(subparsers) + cli._add_subparser_fs_link(subparsers) assert subparsers.choices[STR.link] @@ -376,35 +376,26 @@ def test__dispatch_config_validate_config_obj(): @mark.parametrize( - "action, funcname", [(STR.copy, "_dispatch_file_copy"), (STR.link, "_dispatch_file_link")] + "action, funcname", + [ + (STR.copy, "_dispatch_fs_copy"), + (STR.link, "_dispatch_fs_link"), + (STR.makedirs, "_dispatch_fs_makedirs"), + ], ) -def test__dispatch_file(action, funcname): +def test__dispatch_fs(action, funcname): args = {STR.action: action} with patch.object(cli, funcname) as func: - cli._dispatch_file(args) + cli._dispatch_fs(args) func.assert_called_once_with(args) -def test__dispatch_file_copy(args_dispatch_file): - args = args_dispatch_file - with patch.object(cli.uwtools.api.file, "copy") as copy: - cli._dispatch_file_copy(args) - copy.assert_called_once_with( - target_dir=args["target_dir"], - config=args["config_file"], - cycle=args["cycle"], - leadtime=args["leadtime"], - keys=args["keys"], - dry_run=args["dry_run"], - stdin_ok=args["stdin_ok"], - ) - - -def test__dispatch_file_link(args_dispatch_file): - args = args_dispatch_file - with patch.object(cli.uwtools.api.file, "link") as link: - cli._dispatch_file_link(args) - link.assert_called_once_with( +@mark.parametrize("action", ["copy", "link", "makedirs"]) +def test__dispatch_fs_action(action, args_dispatch_fs): + args = args_dispatch_fs + with patch.object(cli.uwtools.api.fs, action) as a: + getattr(cli, f"_dispatch_fs_{action}")(args) + a.assert_called_once_with( target_dir=args["target_dir"], config=args["config_file"], cycle=args["cycle"], diff --git a/src/uwtools/tests/test_file.py b/src/uwtools/tests/test_fs.py similarity index 69% rename from src/uwtools/tests/test_file.py rename to src/uwtools/tests/test_fs.py index 9e3a94b3a..b83a4fbdf 100644 --- a/src/uwtools/tests/test_file.py +++ b/src/uwtools/tests/test_fs.py @@ -1,12 +1,16 @@ -# pylint: disable=missing-function-docstring,protected-access,redefined-outer-name +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=redefined-outer-name import iotaa import yaml from pytest import fixture, mark, raises -from uwtools import file +from uwtools import fs from uwtools.exceptions import UWConfigError +# Fixtures + @fixture def assets(tmp_path): @@ -25,13 +29,32 @@ def assets(tmp_path): return dstdir, cfgdict, cfgfile +# Helpers + + +class ConcreteStager(fs.Stager): + def _validate(self): + pass + + @property + def _dst_paths(self): + return list(self._config.keys()) + + @property + def _schema(self): + return "some-schema" + + +# Tests + + @mark.parametrize("source", ("dict", "file")) def test_Copier(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() - file.Copier(target_dir=dstdir, config=config, keys=["a", "b"]).go() + fs.Copier(target_dir=dstdir, config=config, keys=["a", "b"]).go() assert (dstdir / "foo").is_file() assert (dstdir / "subdir" / "bar").is_file() @@ -40,7 +63,7 @@ def test_Copier_config_file_dry_run(assets): dstdir, cfgdict, _ = assets assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() - file.Copier(target_dir=dstdir, config=cfgdict, keys=["a", "b"], dry_run=True).go() + fs.Copier(target_dir=dstdir, config=cfgdict, keys=["a", "b"], dry_run=True).go() assert not (dstdir / "foo").exists() assert not (dstdir / "subdir" / "bar").exists() iotaa.dryrun(False) @@ -50,51 +73,49 @@ def test_Copier_no_targetdir_abspath_pass(assets): dstdir, cfgdict, _ = assets old = cfgdict["a"]["b"] cfgdict = {str(dstdir / "foo"): old["foo"], str(dstdir / "bar"): old["subdir/bar"]} - assets = file.Copier(config=cfgdict).go() + assets = fs.Copier(config=cfgdict).go() assert all(asset.ready() for asset in assets) # type: ignore def test_Copier_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: - file.Copier(config=cfgdict, keys=["a", "b"]).go() + fs.Copier(config=cfgdict, keys=["a", "b"]).go() errmsg = "Relative path '%s' requires the target directory to be specified" assert errmsg % "foo" in str(e.value) @mark.parametrize("source", ("dict", "file")) -def test_Linker(assets, source): +def test_FilerStager(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile - assert not (dstdir / "foo").exists() - assert not (dstdir / "subdir" / "bar").exists() - file.Linker(target_dir=dstdir, config=config, keys=["a", "b"]).go() - assert (dstdir / "foo").is_symlink() - assert (dstdir / "subdir" / "bar").is_symlink() + assert fs.FileStager(target_dir=dstdir, config=config, keys=["a", "b"]) @mark.parametrize("source", ("dict", "file")) -def test_Stager(assets, source): +def test_Linker(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile - stager = file.Stager(target_dir=dstdir, config=config, keys=["a", "b"]) - assert set(stager._file_map.keys()) == {"foo", "subdir/bar"} - assert stager._validate() is True + assert not (dstdir / "foo").exists() + assert not (dstdir / "subdir" / "bar").exists() + fs.Linker(target_dir=dstdir, config=config, keys=["a", "b"]).go() + assert (dstdir / "foo").is_symlink() + assert (dstdir / "subdir" / "bar").is_symlink() @mark.parametrize("source", ("dict", "file")) -def test_Stager_bad_key(assets, source): +def test_Stager__config_block_fail_bad_keypath(assets, source): dstdir, cfgdict, cfgfile = assets config = cfgdict if source == "dict" else cfgfile with raises(UWConfigError) as e: - file.Stager(target_dir=dstdir, config=config, keys=["a", "x"]) + ConcreteStager(target_dir=dstdir, config=config, keys=["a", "x"]) assert str(e.value) == "Failed following YAML key(s): a -> x" @mark.parametrize("val", [None, True, False, "str", 88, 3.14, [], tuple()]) -def test_Stager_empty_val(assets, val): +def test_Stager__config_block_fails_bad_type(assets, val): dstdir, cfgdict, _ = assets cfgdict["a"]["b"] = val with raises(UWConfigError) as e: - file.Stager(target_dir=dstdir, config=cfgdict, keys=["a", "b"]) - assert str(e.value) == "No file map found at key path: a -> b" + ConcreteStager(target_dir=dstdir, config=cfgdict, keys=["a", "b"]) + assert str(e.value) == "Expected block not found at key path: a -> b" diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index d270d5cc2..ee0dd3043 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -739,7 +739,7 @@ def test_schema_execution_serial(): # files-to-stage -def test_schema_files_to_stage(): +def test_schema_stage_files(): errors = schema_validator("files-to-stage") # The input must be an dict: assert "is not of type 'object'\n" in errors([]) @@ -1212,6 +1212,21 @@ def test_schema_make_solo_mosaic_rundir(make_solo_mosaic_prop): assert "88 is not of type 'string'\n" in errors(88) +# makedirs + + +def test_schema_makedirs(): + errors = schema_validator("makedirs") + # The input must be an dict: + assert "is not of type 'object'\n" in errors([]) + # Basic correctness: + assert not errors({"makedirs": ["/path/to/dir1", "/path/to/dir2"]}) + # An empty array is not allowed: + assert "[] should be non-empty" in errors({"makedirs": []}) + # Non-string values are not allowed: + assert "True is not of type 'string'\n" in errors({"makedirs": [True]}) + + # mpas diff --git a/src/uwtools/tests/utils/test_tasks.py b/src/uwtools/tests/utils/test_tasks.py index f663957aa..aa606ed09 100644 --- a/src/uwtools/tests/utils/test_tasks.py +++ b/src/uwtools/tests/utils/test_tasks.py @@ -16,6 +16,13 @@ def ready(taskval): # Tests +def test_tasks_directory(tmp_path): + p = tmp_path / "foo" / "bar" + assert not p.is_dir() + assert ready(tasks.directory(path=p)) + assert p.is_dir() + + def test_tasks_executable(tmp_path): p = tmp_path / "program" # Ensure that only our temp directory is on the path: diff --git a/src/uwtools/utils/tasks.py b/src/uwtools/utils/tasks.py index 8a8555204..eef1c9deb 100644 --- a/src/uwtools/utils/tasks.py +++ b/src/uwtools/utils/tasks.py @@ -10,6 +10,19 @@ from iotaa import asset, external, task +@task +def directory(path: Path): + """ + A filesystem directory. + + :param path: Path to the directory. + """ + yield "Directory %s" % path + yield asset(path, path.is_dir) + yield None + path.mkdir(parents=True, exist_ok=True) + + @external def executable(program: Union[Path, str]): """