From 103fb8b0f73208082ace0ac7ab11a586972f512d Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Wed, 19 Aug 2020 13:48:13 -0700 Subject: [PATCH] Revert "Master (#2)" This reverts commit 1d7495b582edb91accb7d6a316c92d8cc9051337. --- .gitlab-ci.yml | 19 --- CHANGELOG.md | 41 ------ Dockerfile | 2 +- Makefile | 2 +- docs/source/faq.rst | 18 --- docs/source/merlin_specification.rst | 1 - docs/source/merlin_variables.rst | 61 +-------- .../modules/hello_world/hello_world.rst | 2 +- .../modules/installation/installation.rst | 6 +- merlin/__init__.py | 4 +- merlin/ascii_art.py | 2 +- merlin/celery.py | 76 ++++++----- merlin/common/__init__.py | 2 +- merlin/common/abstracts/__init__.py | 2 +- merlin/common/abstracts/enums/__init__.py | 2 +- merlin/common/openfilelist.py | 2 +- merlin/common/opennpylib.py | 2 +- merlin/common/sample_index.py | 2 +- merlin/common/sample_index_factory.py | 2 +- merlin/common/security/__init__.py | 2 +- merlin/common/security/encrypt.py | 2 +- .../security/encrypt_backend_traffic.py | 2 +- merlin/common/tasks.py | 2 +- merlin/common/util_sampling.py | 2 +- merlin/config/__init__.py | 2 +- merlin/config/broker.py | 2 +- merlin/config/celeryconfig.py | 29 ---- merlin/config/configfile.py | 12 +- merlin/config/results_backend.py | 2 +- merlin/data/celery/__init__.py | 2 +- merlin/data/celery/app.yaml | 5 +- merlin/data/celery/app_redis.yaml | 5 +- merlin/display.py | 35 +---- merlin/examples/__init__.py | 2 +- .../examples/dev_workflows/full_format.yaml | 7 +- merlin/examples/examples.py | 2 +- merlin/examples/generator.py | 2 +- .../workflows/feature_demo/scripts/pgen.py | 14 -- .../examples/workflows/flux/flux_local.yaml | 3 +- merlin/examples/workflows/flux/flux_par.yaml | 3 +- .../workflows/flux/flux_par_restart.yaml | 4 +- merlin/examples/workflows/flux/flux_test.yaml | 2 +- merlin/examples/workflows/flux/paper.yaml | 2 +- .../workflows/flux/scripts/flux_info.py | 100 ++++---------- merlin/exceptions/__init__.py | 2 +- merlin/log_formatter.py | 2 +- merlin/main.py | 30 +--- merlin/merlin_templates.py | 2 +- merlin/router.py | 2 +- merlin/spec/__init__.py | 2 +- merlin/spec/all_keys.py | 2 +- merlin/spec/defaults.py | 2 +- merlin/spec/expansion.py | 54 ++------ merlin/spec/specification.py | 128 ++++-------------- merlin/study/__init__.py | 2 +- merlin/study/batch.py | 2 +- merlin/study/celeryadapter.py | 5 +- merlin/study/dag.py | 2 +- merlin/study/script_adapter.py | 2 +- merlin/study/step.py | 2 +- merlin/study/study.py | 95 +++---------- merlin/utils.py | 2 +- setup.py | 2 +- tests/integration/run_tests.py | 36 +---- tests/study/test_study.py | 6 +- 65 files changed, 191 insertions(+), 684 deletions(-) delete mode 100644 .gitlab-ci.yml delete mode 100644 merlin/config/celeryconfig.py delete mode 100644 merlin/examples/workflows/feature_demo/scripts/pgen.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 9403886e3..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,19 +0,0 @@ -image: python:3.8-slim-buster - -job1: - script: - - python3 -m venv venv - - source venv/bin/activate - - pip3 install --upgrade pip - - pip3 install -r requirements.txt - - pip3 install -r requirements/dev.txt - - pip3 install -r merlin/examples/workflows/feature_demo/requirements.txt - - pip3 install -e . - - pip3 install --upgrade sphinx - - merlin config - - - merlin stop-workers - - - python3 -m pytest tests/ - - python3 tests/integration/run_tests.py --verbose --local - diff --git a/CHANGELOG.md b/CHANGELOG.md index 9afa89828..dde43211d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,47 +4,6 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.7.3] - -### Fixed -- Completed 1.7.2 fix for `merlin run-workers`. - -## [1.7.2] - -### Fixed -- Fatal bug triggered by a spec missing the `env` or `global.parameters` sections. - -## [1.7.1] - -### Added -- When using the `--samplesfile` flag, the samples file is now copied to `merlin_info` for - provenance. - -### Fixed -- Exceptions in connection check sub-process will now be caught. - -## [1.7.0] - -### Added -- The ability to override any value of the celery configuration thru `app.yaml` in `celery.override`. -- Support and faq entry for `pgen` with `merlin run --pgen` and optional `--parg`. -- Documentation on `level_max_dirs`. -- Easier-to-read provenance specs. -- Documentation on the new 3 types of provenance spec. - -### Fixed -- Flux test example data collection for new versions of flux. -- Fixed Docker ubuntu version. -- Removed expansion of env variables in shell sections (`cmd` and `restart`) of provenance - specs. This allows the shell command itself to expand environment variables, and gives - users greater flexibility. -- Allowed environment variables to be properly expanded in study `description.name`. -- Tilde (~) now properly expands as part of a path in non-shell sections. -- The rediss cert_reqs keyword was changed to ssl_cert_reqs. - -### Changed -- Updated tutorial redis version to 6.0.5. - ## [1.6.2] ### Added diff --git a/Dockerfile b/Dockerfile index cde72bf9a..965ebb029 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu LABEL maintainer="Joe Koning koning1@llnl.gov" ARG USER=merlinu diff --git a/Makefile b/Makefile index 7925b2459..05fb3dccb 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/docs/source/faq.rst b/docs/source/faq.rst index dd848fada..0ee952981 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -377,24 +377,6 @@ num resource set: Number of resource sets launch_distribution : The distribution of resources (default: plane:{procs/nodes}) lsf: Verbatim flags only for the lsf parallel launch (jsrun ... ) -What is level_max_dirs? -~~~~~~~~~~~~~~~~~~~~~~~ -``level_max_dirs`` is an optional field that goes under the ``merlin.samples`` section -of a yaml spec. It caps the number of sample directories that can be generated -at a single level of a study's sample hierarchy. This is useful for getting around -filesystem constraints when working with massive amounts of data. - -Defaults to 25. - -What is pgen? -~~~~~~~~~~~~~ -``pgen`` stands for "parameter generator". It's a way to override the parameters in the -``global.parameters`` spec section, instead generating them programatically with a python script. -Merlin offers the same pgen functionality as Maestro. - -See `this guide `_ for details on using ``pgen``. -It's a Maestro doc, but the exact same flags can be used in conjunction with ``merlin run``. - Where can I learn more about merlin? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Check out `our paper `_ on arXiv. diff --git a/docs/source/merlin_specification.rst b/docs/source/merlin_specification.rst index 062a5b90a..30c66b5f2 100644 --- a/docs/source/merlin_specification.rst +++ b/docs/source/merlin_specification.rst @@ -290,4 +290,3 @@ see :doc:`./merlin_variables`. generate: cmd: | python $(SPECROOT)/make_samples.py -dims 2 -n 10 -outfile=$(INPUT_PATH)/samples.npy "[(1.3, 1.3, 'linear'), (3.3, 3.3, 'linear')]" - level_max_dirs: 25 diff --git a/docs/source/merlin_variables.rst b/docs/source/merlin_variables.rst index ad5bb376b..050846b21 100644 --- a/docs/source/merlin_variables.rst +++ b/docs/source/merlin_variables.rst @@ -21,10 +21,8 @@ The directory structure of merlin output looks like this: OUTPUT_PATH MERLIN_WORKSPACE MERLIN_INFO - .orig.yaml - .partial.yaml - .expanded.yaml - .workspace + + .workspace WORKSPACE @@ -54,7 +52,7 @@ Reserved variables - The workspace directory for a single step. - ``$(OUTPUT_PATH)/ensemble_name_$(MERLIN_TIMESTAMP)/step_name/`` * - ``$(MERLIN_INFO)`` - - Directory within ``MERLIN_WORKSPACE`` that holds the provenance specs and sample generation results. + - Directory within ``MERLIN_WORKSPACE`` that holds a provenance spec. Commonly used to hold ``samples.npy``. - ``$(MERLIN_WORKSPACE)/merlin_info/`` * - ``$(MERLIN_SAMPLE_ID)`` @@ -93,68 +91,17 @@ Variables defined by a specification file in the ``env`` section, as in this exa env: variables: - ID: 42 EXAMPLE_VAR: hello -As long as they're defined in order, you can nest user variables like this: -.. code-block:: yaml - - env: - variables: - EXAMPLE_VAR: hello - WORKER_NAME: $(EXAMPLE_VAR)_worker - -Like all other Merlin variables, user variables may be used anywhere (as a yaml key or value) within a specification as below: +Like all other Merlin variables, these may be used within specification steps as below: .. code-block:: yaml cmd: echo "$(EXAMPLE_VAR), world!" - ... - $(WORKER_NAME): - args: ... - -If you want to programmatically define the study name, you can include variables -in the ``description.name`` field as long as it makes a valid filename: - -.. code-block:: yaml - - description: - name: my_$(EXAMPLE_VAR)_study_$(ID) - description: example of programmatic study name - -The above would produce a study called ``my_hello_study_42``. -Environment variables ---------------------- -Merlin expands Unix environment variables for you. The values of the user variables below would be expanded: -.. code-block:: yaml - - env: - variables: - MY_HOME: ~/ - MY_PATH: $PATH - USERNAME: ${USER} - -However, Merlin leaves environment variables found in shell scripts (think ``cmd`` and ``restart``) alone. -So this step: - -.. code-block:: yaml - - - name: step1 - description: an example - run: - cmd: echo $PATH ; echo $(MY_PATH) - -...would be expanded as: - -.. code-block:: yaml - - name: step1 - description: an example - run: - cmd: echo $PATH ; echo /an/example/:/path/string/ Step return variables ----------------------------------- diff --git a/docs/source/modules/hello_world/hello_world.rst b/docs/source/modules/hello_world/hello_world.rst index 31ea31976..d024d21b2 100644 --- a/docs/source/modules/hello_world/hello_world.rst +++ b/docs/source/modules/hello_world/hello_world.rst @@ -188,7 +188,7 @@ The whole file tree looks like this: A lot of stuff, right? Here's what it means: -* The 3 yaml files inside ``merlin_info/`` are called the provenance specs. They are copies of the original spec that was run, some showing under-the-hood variable expansions. +* The yaml file inside ``merlin_info/`` is called the provenance spec. It's a copy of the original spec that was run. * ``MERLIN_FINISHED`` files indicate that the step ran successfully. diff --git a/docs/source/modules/installation/installation.rst b/docs/source/modules/installation/installation.rst index dabc4f807..2d03597f1 100644 --- a/docs/source/modules/installation/installation.rst +++ b/docs/source/modules/installation/installation.rst @@ -94,7 +94,7 @@ can also be used for the celery broker. This method will be called local-redis. .. code-block:: bash # Download redis - wget http://download.redis.io/releases/redis-6.0.5.tar.gz + wget http://download.redis.io/releases/redis-5.0.7.tar.gz # Untar tar xvf redis*.tar.gz @@ -361,8 +361,8 @@ server and python. mkdir -p ${HOME}/merlinu/cert_redis cp result/* ${HOME}/merlinu/cert_redis - -The configuration below does not use client +The ``redis:6.0-rc2`` docker service is exchanged for the previous +``redis:latest`` service. The configuration below does not use client verification ``--tls-auth-clients no`` so the ssl files do not need to be defined as shown in the :ref:`broker_redis_ssl` section. diff --git a/merlin/__init__.py b/merlin/__init__.py index 2e5e76f16..d2234f10c 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -38,7 +38,7 @@ import sys -__version__ = "1.7.3" +__version__ = "1.6.2" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/merlin/ascii_art.py b/merlin/ascii_art.py index 2ae5be423..7d76d0556 100644 --- a/merlin/ascii_art.py +++ b/merlin/ascii_art.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/celery.py b/merlin/celery.py index 94ecd69e1..51fab12da 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -40,10 +40,10 @@ from celery.signals import worker_process_init import merlin.common.security.encrypt_backend_traffic -from merlin.config import broker, celeryconfig, results_backend +from merlin.config import broker, results_backend from merlin.config.configfile import CONFIG +from merlin.log_formatter import FORMATS from merlin.router import route_for_task -from merlin.utils import nested_namespace_to_dicts LOG = logging.getLogger(__name__) @@ -54,59 +54,65 @@ results_ssl = False try: BROKER_URI = broker.get_connection_string() - LOG.debug(f"broker: {broker.get_connection_string(include_password=False)}") + LOG.info(f"broker: {broker.get_connection_string(include_password=False)}") broker_ssl = broker.get_ssl_config() - LOG.debug(f"broker_ssl = {broker_ssl}") + LOG.info(f"broker_ssl = {broker_ssl}") RESULTS_BACKEND_URI = results_backend.get_connection_string() results_ssl = results_backend.get_ssl_config(celery_check=True) - LOG.debug( + LOG.info( f"results: {results_backend.get_connection_string(include_password=False)}" ) - LOG.debug(f"results: redis_backed_use_ssl = {results_ssl}") + LOG.info(f"results: redis_backed_use_ssl = {results_ssl}") except ValueError: # These variables won't be set if running with '--local'. BROKER_URI = None RESULTS_BACKEND_URI = None -# initialize app with essential properties + app = Celery( "merlin", broker=BROKER_URI, backend=RESULTS_BACKEND_URI, broker_use_ssl=broker_ssl, redis_backend_use_ssl=results_ssl, - task_routes=(route_for_task,), ) -# load merlin config defaults -app.conf.update(**celeryconfig.DICT) - -# load config overrides from app.yaml -if ( - (not hasattr(CONFIG.celery, "override")) - or (CONFIG.celery.override is None) - or (len(nested_namespace_to_dicts(CONFIG.celery.override)) == 0) -): - LOG.debug("Skipping celery config override; 'celery.override' field is empty.") -else: - override_dict = nested_namespace_to_dicts(CONFIG.celery.override) - override_str = "" - i = 0 - for k, v in override_dict.items(): - if k not in str(app.conf.__dict__): - raise ValueError(f"'{k}' is not a celery configuration.") - override_str += f"\t{k}:\t{v}" - if i != len(override_dict) - 1: - override_str += "\n" - i += 1 - LOG.info( - f"Overriding default celery config with 'celery.override' in 'app.yaml':\n{override_str}" - ) - app.conf.update(**override_dict) -# auto-discover tasks +app.conf.update( + task_serializer="pickle", accept_content=["pickle"], result_serializer="pickle" +) + app.autodiscover_tasks(["merlin.common"]) +app.conf.update( + task_acks_late=True, + task_reject_on_worker_lost=True, + task_publish_retry_policy={ + "interval_start": 10, + "interval_step": 10, + "interval_max": 60, + }, + redis_max_connections=100000, +) + +# Set a timeout to acknowledge a task before it's available to grab +# again (default 24 hours). +app.conf.broker_transport_options = { + "visibility_timeout_seconds": CONFIG.celery.visibility_timeout_seconds, + "max_connections": 100, +} + +app.conf.update(broker_pool_limit=0) + +# Task routing: call our default queue merlin +app.conf.task_routes = (route_for_task,) +app.conf.task_default_queue = "merlin" + +# Log formatting +app.conf.worker_log_color = True +app.conf.worker_log_format = FORMATS["DEFAULT"] +app.conf.worker_task_log_format = FORMATS["WORKER"] + @worker_process_init.connect() def setup(**kwargs): diff --git a/merlin/common/__init__.py b/merlin/common/__init__.py index c16ed892b..9c7906d9e 100644 --- a/merlin/common/__init__.py +++ b/merlin/common/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/__init__.py b/merlin/common/abstracts/__init__.py index c16ed892b..9c7906d9e 100644 --- a/merlin/common/abstracts/__init__.py +++ b/merlin/common/abstracts/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/abstracts/enums/__init__.py b/merlin/common/abstracts/enums/__init__.py index 9145e0877..280fc77b8 100644 --- a/merlin/common/abstracts/enums/__init__.py +++ b/merlin/common/abstracts/enums/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/openfilelist.py b/merlin/common/openfilelist.py index eac2d8808..8e11fc346 100644 --- a/merlin/common/openfilelist.py +++ b/merlin/common/openfilelist.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/opennpylib.py b/merlin/common/opennpylib.py index d403d07b0..b0d2d7868 100644 --- a/merlin/common/opennpylib.py +++ b/merlin/common/opennpylib.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index.py b/merlin/common/sample_index.py index e4bbd56ea..b10759073 100644 --- a/merlin/common/sample_index.py +++ b/merlin/common/sample_index.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index af0441aa4..d0f887173 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/__init__.py b/merlin/common/security/__init__.py index c16ed892b..9c7906d9e 100644 --- a/merlin/common/security/__init__.py +++ b/merlin/common/security/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt.py b/merlin/common/security/encrypt.py index d8d52a9e4..c85f01cbc 100644 --- a/merlin/common/security/encrypt.py +++ b/merlin/common/security/encrypt.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/security/encrypt_backend_traffic.py b/merlin/common/security/encrypt_backend_traffic.py index b469aac77..6df74b7dc 100644 --- a/merlin/common/security/encrypt_backend_traffic.py +++ b/merlin/common/security/encrypt_backend_traffic.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index a8ed9e31d..01b0a8726 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/common/util_sampling.py b/merlin/common/util_sampling.py index a7b0397db..0830240fb 100644 --- a/merlin/common/util_sampling.py +++ b/merlin/common/util_sampling.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/__init__.py b/merlin/config/__init__.py index 749ffd5b8..333c83f9b 100644 --- a/merlin/config/__init__.py +++ b/merlin/config/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/broker.py b/merlin/config/broker.py index cc45cbaa9..8b9e08ed6 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/config/celeryconfig.py b/merlin/config/celeryconfig.py deleted file mode 100644 index bf58602d5..000000000 --- a/merlin/config/celeryconfig.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Default celery configuration for merlin -""" - -from merlin.log_formatter import FORMATS - - -DICT = { - "task_serializer": "pickle", - "accept_content": ["pickle"], - "result_serializer": "pickle", - "task_acks_late": True, - "task_reject_on_worker_lost": True, - "task_publish_retry_policy": { - "interval_start": 10, - "interval_step": 10, - "interval_max": 60, - }, - "redis_max_connections": 100000, - "broker_transport_options": { - "visibility_timeout": 60 * 60 * 24, - "max_connections": 100, - }, - "broker_pool_limit": 0, - "task_default_queue": "merlin", - "worker_log_color": True, - "worker_log_format": FORMATS["DEFAULT"], - "worker_task_log_format": FORMATS["WORKER"], -} diff --git a/merlin/config/configfile.py b/merlin/config/configfile.py index 7b3780d30..313025591 100644 --- a/merlin/config/configfile.py +++ b/merlin/config/configfile.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -129,20 +129,21 @@ def get_config(path): return config -def load_default_celery(config): +def load_default_timeout(config): + seconds = 60 * 60 * 24 try: config["celery"] except KeyError: config["celery"] = {} try: - config["celery"]["override"] + config["celery"]["visibility_timeout_seconds"] except KeyError: - config["celery"]["override"] = None + config["celery"]["visibility_timeout_seconds"] = seconds def load_defaults(config): load_default_user_names(config) - load_default_celery(config) + load_default_timeout(config) def is_debug(): @@ -245,7 +246,6 @@ def get_ssl_entries(server_type, server_name, server_config, cert_path): "keyfile": "ssl_keyfile", "certfile": "ssl_certfile", "ca_certs": "ssl_ca_certs", - "cert_reqs": "ssl_cert_reqs", } # The mysql server requires key names with ssl_ and different var names diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index e41d9a9ce..caccc9334 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/__init__.py b/merlin/data/celery/__init__.py index c16ed892b..9c7906d9e 100644 --- a/merlin/data/celery/__init__.py +++ b/merlin/data/celery/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/data/celery/app.yaml b/merlin/data/celery/app.yaml index 3b37c09fb..e4631b75a 100644 --- a/merlin/data/celery/app.yaml +++ b/merlin/data/celery/app.yaml @@ -1,8 +1,5 @@ celery: - # see Celery configuration options - # https://docs.celeryproject.org/en/stable/userguide/configuration.html - override: - visibility_timeout: 86400 + visibility_timeout_seconds: 86400 broker: # can be redis, redis+sock, or rabbitmq diff --git a/merlin/data/celery/app_redis.yaml b/merlin/data/celery/app_redis.yaml index 1b08313ad..ecee7d398 100644 --- a/merlin/data/celery/app_redis.yaml +++ b/merlin/data/celery/app_redis.yaml @@ -1,8 +1,5 @@ celery: - # see Celery configuration options - # https://docs.celeryproject.org/en/stable/userguide/configuration.html - override: - visibility_timeout: 86400 + visibility_timeout_seconds: 86400 broker: # can be redis, redis+sock, or rabbitmq diff --git a/merlin/display.py b/merlin/display.py index c8dd77806..29eb97dfd 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -34,8 +34,7 @@ import pprint import subprocess import time -import traceback -from multiprocessing import Pipe, Process +from multiprocessing import Process from kombu import Connection from tabulate import tabulate @@ -45,28 +44,6 @@ from merlin.config.configfile import default_config_info -class ConnProcess(Process): - def __init__(self, *args, **kwargs): - Process.__init__(self, *args, **kwargs) - self._pconn, self._cconn = Pipe() - self._exception = None - - def run(self): - try: - Process.run(self) - self._cconn.send(None) - except Exception as e: - tb = traceback.format_exc() - self._cconn.send((e, tb)) - # raise e # You can still rise this exception if you need to - - @property - def exception(self): - if self._pconn.poll(): - self._exception = self._pconn.recv() - return self._exception - - def check_server_access(sconf): servers = ["broker server", "results server"] @@ -80,7 +57,7 @@ def check_server_access(sconf): if s in sconf: try: conn = Connection(sconf[s]) - conn_check = ConnProcess(target=conn.connect) + conn_check = Process(target=conn.connect) conn_check.start() counter = 0 while conn_check.is_alive(): @@ -92,14 +69,10 @@ def check_server_access(sconf): f"Connection was killed due to timeout ({connect_timeout}s)" ) conn.release() - if conn_check.exception: - error, traceback = conn_check.exception - raise error + print(f"{s} connection: OK") except Exception as e: print(f"{s} connection: Error") excpts[s] = e - else: - print(f"{s} connection: OK") if excpts: print("\nExceptions:") diff --git a/merlin/examples/__init__.py b/merlin/examples/__init__.py index c16ed892b..9c7906d9e 100644 --- a/merlin/examples/__init__.py +++ b/merlin/examples/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/dev_workflows/full_format.yaml b/merlin/examples/dev_workflows/full_format.yaml index 0fd6ab3fc..981cb2122 100644 --- a/merlin/examples/dev_workflows/full_format.yaml +++ b/merlin/examples/dev_workflows/full_format.yaml @@ -10,7 +10,6 @@ env: LABEL: label sources: - source1 - - source2 dependencies: paths: - name: name1 @@ -74,10 +73,10 @@ study: pre: string depends: [step] -global.parameters: - PARAM: +global.params: + - PARAM: values: [val1, val2] - label: LABEL.%% + label: label merlin: resources: diff --git a/merlin/examples/examples.py b/merlin/examples/examples.py index 34bceb9ba..cc5b83920 100644 --- a/merlin/examples/examples.py +++ b/merlin/examples/examples.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/generator.py b/merlin/examples/generator.py index 7336fa860..e90bbdcb0 100644 --- a/merlin/examples/generator.py +++ b/merlin/examples/generator.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/examples/workflows/feature_demo/scripts/pgen.py b/merlin/examples/workflows/feature_demo/scripts/pgen.py deleted file mode 100644 index 99b9b105d..000000000 --- a/merlin/examples/workflows/feature_demo/scripts/pgen.py +++ /dev/null @@ -1,14 +0,0 @@ -from maestrowf.datastructures.core import ParameterGenerator - - -def get_custom_generator(env, **kwargs): - p_gen = ParameterGenerator() - params = { - "X2": {"values": [1 / i for i in range(3, 6)], "label": "X2.%%"}, - "N_NEW": {"values": [2 ** i for i in range(1, 4)], "label": "N_NEW.%%"}, - } - - for key, value in params.items(): - p_gen.add_parameter(key, value["values"], value["label"]) - - return p_gen diff --git a/merlin/examples/workflows/flux/flux_local.yaml b/merlin/examples/workflows/flux/flux_local.yaml index 3cd50cd46..322f9bfb3 100644 --- a/merlin/examples/workflows/flux/flux_local.yaml +++ b/merlin/examples/workflows/flux/flux_local.yaml @@ -10,7 +10,6 @@ batch: env: variables: - OUTPUT_PATH: ./studies N_SAMPLES: 10 study: @@ -46,7 +45,7 @@ study: name: data run: cmd: | - $(SPECROOT)/scripts/flux_info.py > flux_timings.out + flux kvs dir lwj.0.0.5 >& flux_kvs.out depends: [runs*] task_queue: flux_local diff --git a/merlin/examples/workflows/flux/flux_par.yaml b/merlin/examples/workflows/flux/flux_par.yaml index d401a0270..2c3ce3c5f 100644 --- a/merlin/examples/workflows/flux/flux_par.yaml +++ b/merlin/examples/workflows/flux/flux_par.yaml @@ -10,7 +10,6 @@ batch: env: variables: - OUTPUT_PATH: ./studies N_SAMPLES: 10 study: @@ -44,7 +43,7 @@ study: name: data run: cmd: | - $(SPECROOT)/scripts/flux_info.py > flux_timings.out + flux kvs dir lwj.0.0.5 >& flux_kvs.out depends: [runs_*] task_queue: flux_par diff --git a/merlin/examples/workflows/flux/flux_par_restart.yaml b/merlin/examples/workflows/flux/flux_par_restart.yaml index adb16f71c..530a84a5e 100644 --- a/merlin/examples/workflows/flux/flux_par_restart.yaml +++ b/merlin/examples/workflows/flux/flux_par_restart.yaml @@ -99,8 +99,8 @@ study: name: data run: cmd: | - $(SPECROOT)/scripts/flux_info.py > flux_timings.out - depends: [runs_*, runs_rs_*, runs_rt_*] + flux kvs dir lwj.0.0.5 >& flux_kvs.out + depends: [runs*, runs_rs*, runs_rt*] task_queue: flux_par global.parameters: diff --git a/merlin/examples/workflows/flux/flux_test.yaml b/merlin/examples/workflows/flux/flux_test.yaml index e0f85795f..1bf05a4cd 100644 --- a/merlin/examples/workflows/flux/flux_test.yaml +++ b/merlin/examples/workflows/flux/flux_test.yaml @@ -27,7 +27,7 @@ study: name: data run: cmd: | - $(SPECROOT)/scripts/flux_info.py > flux_timings.out + flux kvs dir lwj.0.0.5 >& flux_kvs.out depends: [runs*] task_queue: flux_test diff --git a/merlin/examples/workflows/flux/paper.yaml b/merlin/examples/workflows/flux/paper.yaml index 61c7681e3..8d040443d 100644 --- a/merlin/examples/workflows/flux/paper.yaml +++ b/merlin/examples/workflows/flux/paper.yaml @@ -52,7 +52,7 @@ study: cmd: | #export PATH=/collab/usr/gapps/merlin/spack/idesign/${SYS_TYPE}/bin:$PATH $(SPECROOT)/scripts/flux_info.py >flux_timings.out - depends: [runs_*] + depends: [runs*] task_queue: flux_paper global.parameters: diff --git a/merlin/examples/workflows/flux/scripts/flux_info.py b/merlin/examples/workflows/flux/scripts/flux_info.py index a2e1efe9b..c6aae56f4 100755 --- a/merlin/examples/workflows/flux/scripts/flux_info.py +++ b/merlin/examples/workflows/flux/scripts/flux_info.py @@ -1,28 +1,15 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python2 """This module will collect information on flux jobs from the live kvs store and output times for each phase. -old: create: The time flux registerd the job starting: The time the job was created running: The time the job was running completing: The time the job started its' completion pahse. complete: The time the job was complete walltime: ? Seems to be 0. - -new: -init: -starting: -shell.init: -shell.start: -complete: -cleanup.start: -cleanup.finish: -done: """ -import json import os -import subprocess import flux from flux import kvs @@ -34,76 +21,37 @@ if fs in os.environ: print("flux start: {0}".format(os.environ[fs])) -try: - kvs.get(f, "lwj") - - for d in kvs.walk("lwj", flux_handle=f): - try: - # print(type(d)) - fdir = "lwj.{0}".format(d[0]) +for d in kvs.walk("lwj", flux_handle=f): + try: + # print(type(d)) + fdir = "lwj.{0}".format(d[0]) - qcreate = "{0}.create-time".format(fdir) - create_time = kvs.get(f, qcreate) + qcreate = "{0}.create-time".format(fdir) + create_time = kvs.get(f, qcreate) - qstart = "{0}.starting-time".format(fdir) - start_time = kvs.get(f, qstart) + qstart = "{0}.starting-time".format(fdir) + start_time = kvs.get(f, qstart) - qrun = "{0}.running-time".format(fdir) - start_time = kvs.get(f, qrun) + qrun = "{0}.running-time".format(fdir) + start_time = kvs.get(f, qrun) - qcomplete = "{0}.complete-time".format(fdir) - complete_time = kvs.get(f, qcomplete) + qcomplete = "{0}.complete-time".format(fdir) + complete_time = kvs.get(f, qcomplete) - qcompleting = "{0}.completing-time".format(fdir) - completing_time = kvs.get(f, qcompleting) + qcompleting = "{0}.completing-time".format(fdir) + completing_time = kvs.get(f, qcompleting) - qwall = "{0}.walltime".format(fdir) - wall_time = kvs.get(f, qwall) + qwall = "{0}.walltime".format(fdir) + wall_time = kvs.get(f, qwall) - print( - f"Job {d[0]}: create: {create_time} start {start_time} run {start_time} completing {completing_time} complete {complete_time} wall {wall_time}" + proto = "Job {0}: create: {1} start {1} run {2} completing {3} complete {4} wall {5}" + print( + proto.format( + d[0], create_time, start_time, completing_time, complete_time, wall_time ) - except BaseException: - pass -except: - top_dir = "job" - - def get_data_dict(key): - kwargs = { - "env": os.environ, - "shell": True, - "universal_newlines": True, - "stdout": subprocess.PIPE, - "stderr": subprocess.PIPE, - } - flux_com = f"flux kvs get {key}" - p = subprocess.Popen(flux_com, **kwargs) - stdout, stderr = p.communicate() - - data = {} - for l in stdout.split("/n"): - for s in l.strip().split(): - if "timestamp" in s: - jstring = s.replace("'", '"') - d = json.loads(jstring) - data[d["name"]] = d["timestamp"] - - return data - - for d in kvs.walk(top_dir, flux_handle=f): - if "exec" in d[0]: - for e in d[2]: - key = ".".join([top_dir, d[0], e]) - - # This is currently not working gives - # json.decoder.JSONDecodeError - # data = kvs.get(f, key) - - data = get_data_dict(key) - - print( - f"Job {d[0]}: init: {data['init']} start {data['shell.start']} complete {data['complete']} done {data['done']} " - ) + ) + except BaseException: + pass # vi: ts=4 sw=4 expandtab diff --git a/merlin/exceptions/__init__.py b/merlin/exceptions/__init__.py index edbc0232c..316a2a6b3 100644 --- a/merlin/exceptions/__init__.py +++ b/merlin/exceptions/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/log_formatter.py b/merlin/log_formatter.py index 7c93229d6..c76a200c5 100644 --- a/merlin/log_formatter.py +++ b/merlin/log_formatter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/main.py b/merlin/main.py index 02bd759c2..a0275cb70 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -163,23 +163,12 @@ def process_run(args): samples_file = None if args.samples_file: samples_file = verify_filepath(args.samples_file) - - # pgen checks - if args.pargs and not args.pgen_file: - raise ValueError( - "Cannot use the 'pargs' parameter without specifying a 'pgen'!" - ) - if args.pgen_file: - verify_filepath(args.pgen_file) - study = MerlinStudy( filepath, override_vars=variables_dict, samples_file=samples_file, dry_run=args.dry, no_errors=args.no_errors, - pgen_file=args.pgen_file, - pargs=args.pargs, ) router.run_task_server(study, args.run_mode) @@ -415,23 +404,6 @@ def setup_argparse(): default=False, help="Flag to ignore some errors for testing.", ) - run.add_argument( - "--pgen", - action="store", - dest="pgen_file", - type=str, - default=None, - help="Provide a pgen file to override global.parameters.", - ) - run.add_argument( - "--pargs", - type=str, - action="append", - default=[], - help="A string that represents a single argument to pass " - "a custom parameter generation function. Reuse '--parg' " - "to pass multiple arguments. [Use with '--pgen']", - ) # merlin restart restart = subparsers.add_parser( diff --git a/merlin/merlin_templates.py b/merlin/merlin_templates.py index a6ad512d5..f15911540 100644 --- a/merlin/merlin_templates.py +++ b/merlin/merlin_templates.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/router.py b/merlin/router.py index 45406227f..e028a9b04 100644 --- a/merlin/router.py +++ b/merlin/router.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index c16ed892b..9c7906d9e 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/all_keys.py b/merlin/spec/all_keys.py index df6b827a2..7ef320bde 100644 --- a/merlin/spec/all_keys.py +++ b/merlin/spec/all_keys.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/defaults.py b/merlin/spec/defaults.py index c520507e1..2de803706 100644 --- a/merlin/spec/defaults.py +++ b/merlin/spec/defaults.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index 404a1222c..4ae544671 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -30,7 +30,6 @@ import logging from collections import ChainMap -from copy import deepcopy from os.path import expanduser, expandvars from merlin.common.abstracts.enums import ReturnCode @@ -76,22 +75,17 @@ def var_ref(string): return f"$({string})" -def expand_line(line, var_dict, env_vars=False): +def expand_line(line, var_dict): """ - Expand one line of text by substituting user variables, - optionally environment variables, as well as variables in 'var_dict'. + Expand one line of text by substituting environment + and user variables, as well as variables in 'var_dict'. """ - if ( - (not contains_token(line)) - and (not contains_shell_ref(line)) - and ("~" not in line) - ): + line = expandvars(expanduser(line)) + if not contains_token(line): return line for key, val in var_dict.items(): if key in line: line = line.replace(var_ref(key), str(val)) - if env_vars: - line = expandvars(expanduser(line)) return line @@ -108,33 +102,6 @@ def expand_by_line(text, var_dict): return result -def expand_env_vars(spec): - """ - Expand environment variables for all sections of a spec, except - for values with the key 'cmd' or 'restart' (these are executable - shell scripts, so environment variable expansion would be redundant). - """ - - def recurse(section): - if section is None: - return section - if isinstance(section, str): - return expandvars(expanduser(section)) - if isinstance(section, dict): - for k, v in section.items(): - if k in ["cmd", "restart"]: - continue - section[k] = recurse(v) - elif isinstance(section, list): - for i, elem in enumerate(deepcopy(section)): - section[i] = recurse(elem) - return section - - for name, section in spec.sections.items(): - setattr(spec, name, recurse(section)) - return spec - - def determine_user_variables(*user_var_dicts): """ Given an arbitrary number of dictionaries, determine them @@ -155,11 +122,7 @@ def determine_user_variables(*user_var_dicts): {'TARGET': 'target_dir', 'PATH': '$(SPECROOT)/target_dir'} """ - # TODO move this logic to specification.py? - try: - all_var_dicts = dict(ChainMap(*user_var_dicts)) - except TypeError: - all_var_dicts = {} + all_var_dicts = dict(ChainMap(*user_var_dicts)) determined_results = {} for key, val in all_var_dicts.items(): if key in RESERVED: @@ -174,7 +137,8 @@ def determine_user_variables(*user_var_dicts): new_val = new_val.replace( var_determined_key, determined_results[determined_key] ) - new_val = expandvars(expanduser(new_val)) + if contains_shell_ref(new_val): + new_val = expandvars(new_val) determined_results[key.upper()] = new_val return determined_results diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 4d8635881..d216d3341 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -43,6 +43,14 @@ from merlin.spec import all_keys, defaults +def represent_none(self, _): + """Allows yaml to dump None as '' instead of 'null'""" + return self.represent_scalar("tag:yaml.org,2002:null", "") + + +yaml.add_representer(type(None), represent_none) + + LOG = logging.getLogger(__name__) @@ -68,36 +76,6 @@ class MerlinSpec(YAMLSpecification): def __init__(self): super(MerlinSpec, self).__init__() - @property - def yaml_sections(self): - """ - Returns a nested dictionary of all sections of the specification - as used in a yaml spec. - """ - return { - "description": self.description, - "batch": self.batch, - "env": self.environment, - "study": self.study, - "global.parameters": self.globals, - "merlin": self.merlin, - } - - @property - def sections(self): - """ - Returns a nested dictionary of all sections of the specification - as referenced by Maestro's YAMLSpecification class. - """ - return { - "description": self.description, - "batch": self.batch, - "environment": self.environment, - "study": self.study, - "globals": self.globals, - "merlin": self.merlin, - } - @classmethod def load_specification(cls, filepath, suppress_warning=True): spec = super(MerlinSpec, cls).load_specification(filepath) @@ -214,77 +192,25 @@ def check_section(section_name, section, all_keys): def dump(self): """ - Dump this MerlinSpec to a pretty yaml string. + Dump this MerlinSpec to a yaml string. """ - tab = " " - list_offset = " " - from copy import deepcopy - - def dict_to_yaml(obj, string, key_stack, newline=True): - if obj is None: - return "" - lvl = len(key_stack) - 1 - if isinstance(obj, str): - split = obj.splitlines() - if len(split) > 1: - obj = "|\n" + tab * (lvl + 1) + ("\n" + tab * (lvl + 1)).join(split) - return obj - if isinstance(obj, bool): - return str(obj).lower() - if (not isinstance(obj, list)) and (not isinstance(obj, dict)): - return obj - if isinstance(obj, list): - n = len(obj) - use_hyphens = key_stack[-1] in ["paths", "sources", "git", "study"] - if not use_hyphens: - string += "[" - else: - string += "\n" - for i, elem in enumerate(obj): - key_stack = deepcopy(key_stack) - key_stack.append("elem") - if use_hyphens: - string += ( - (lvl + 1) * tab - + "- " - + str(dict_to_yaml(elem, "", key_stack)) - + "\n" - ) - else: - string += str( - dict_to_yaml(elem, "", key_stack, newline=(i != 0)) - ) - if n > 1 and i != len(obj) - 1: - string += ", " - key_stack.pop() - if not use_hyphens: - string += "]" - if isinstance(obj, dict): - if len(key_stack) > 0 and key_stack[-1] != "elem": - string += "\n" - i = 0 - for k, v in obj.items(): - key_stack = deepcopy(key_stack) - key_stack.append(k) - if len(key_stack) > 1 and key_stack[-2] == "elem" and i == 0: - # string += (tab * (lvl - 1)) - string += "" - elif "elem" in key_stack: - string += list_offset + (tab * lvl) - else: - string += tab * (lvl + 1) - string += str(k) + ": " + str(dict_to_yaml(v, "", key_stack)) + "\n" - key_stack.pop() - i += 1 - return string - - result = dict_to_yaml(self.yaml_sections, "", []) - while "\n\n\n" in result: - result = result.replace("\n\n\n", "\n\n") - try: - yaml.safe_load(result) - except BaseException as e: - raise ValueError(f"Error parsing provenance spec:\n{e}") + description = {"description": self.description} + batch = {"batch": self.batch} + env = {"env": self.environment} + study = {"study": self.study} + _global = {"global.parameters": self.globals} + merlin = {"merlin": self.merlin} + + result = "" + result += ( + yaml.dump(description, default_flow_style=False, sort_keys=False) + "\n" + ) + result += yaml.dump(batch, default_flow_style=False, sort_keys=False) + "\n" + result += yaml.dump(env, default_flow_style=False, sort_keys=False) + "\n" + result += yaml.dump(study, default_flow_style=False, sort_keys=False) + "\n" + result += yaml.dump(_global, default_flow_style=False, sort_keys=False) + "\n" + result += yaml.dump(merlin, default_flow_style=False, sort_keys=False) + return result def get_task_queues(self): diff --git a/merlin/study/__init__.py b/merlin/study/__init__.py index c16ed892b..9c7906d9e 100644 --- a/merlin/study/__init__.py +++ b/merlin/study/__init__.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/batch.py b/merlin/study/batch.py index 655bf44f2..edfa6796f 100644 --- a/merlin/study/batch.py +++ b/merlin/study/batch.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index ba874869a..9f4997062 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -214,9 +214,6 @@ def start_celery_workers(spec, steps, celery_args, just_return_command): yenv = None if senv: yenv = get_yaml_var(senv, "variables", {}) - # TODO fix logic in specification.py - if yenv is None: - yenv = {} for k, v in yenv.items(): spenv[str(k)] = str(v) # For expandvars diff --git a/merlin/study/dag.py b/merlin/study/dag.py index 0fa4770bc..7cb48a33c 100644 --- a/merlin/study/dag.py +++ b/merlin/study/dag.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/script_adapter.py b/merlin/study/script_adapter.py index 1d3a56b47..87e2f2080 100644 --- a/merlin/study/script_adapter.py +++ b/merlin/study/script_adapter.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/step.py b/merlin/study/step.py index cfcf2a7c4..95ce7915a 100644 --- a/merlin/study/step.py +++ b/merlin/study/step.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/merlin/study/study.py b/merlin/study/study.py index d26b9fa58..5dbff80e6 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -38,26 +38,14 @@ from cached_property import cached_property from maestrowf.datastructures.core import Study -from maestrowf.maestro import load_parameter_generator -from maestrowf.utils import create_dictionary from merlin.common.abstracts.enums import ReturnCode from merlin.spec import defaults -from merlin.spec.expansion import ( - determine_user_variables, - expand_by_line, - expand_env_vars, - expand_line, -) +from merlin.spec.expansion import determine_user_variables, expand_by_line, expand_line from merlin.spec.override import dump_with_overrides, error_override_vars from merlin.spec.specification import MerlinSpec from merlin.study.dag import DAG -from merlin.utils import ( - contains_shell_ref, - contains_token, - get_flux_cmd, - load_array_file, -) +from merlin.utils import contains_token, get_flux_cmd, load_array_file LOG = logging.getLogger(__name__) @@ -86,8 +74,6 @@ def __init__( samples_file=None, dry_run=False, no_errors=False, - pgen_file=None, - pargs=None, ): self.original_spec = MerlinSpec.load_specification(filepath) self.override_vars = override_vars @@ -118,17 +104,10 @@ def __init__( "MERLIN_HARD_FAIL": str(int(ReturnCode.HARD_FAIL)), "MERLIN_RETRY": str(int(ReturnCode.RETRY)), } - - self.pgen_file = pgen_file - self.pargs = pargs - self.dag = None self.load_dag() def write_original_spec(self, filename): - """ - Copy the original spec into merlin_info/ as '.orig.yaml'. - """ spec_name = os.path.join(self.info, filename + ".orig.yaml") shutil.copyfile(self.original_spec.path, spec_name) @@ -179,8 +158,7 @@ def get_expanded_spec(self): # expand reserved words new_spec_text = expand_by_line(new_spec_text, self.special_vars) - result = MerlinSpec.load_spec_from_string(new_spec_text) - return expand_env_vars(result) + return MerlinSpec.load_spec_from_string(new_spec_text) @property def samples(self): @@ -289,7 +267,7 @@ def output_path(self): ): output_path = str(self.override_vars["OUTPUT_PATH"]) - output_path = expand_line(output_path, self.user_vars, env_vars=True) + output_path = expand_line(output_path, self.user_vars) output_path = os.path.abspath(output_path) if not os.path.isdir(output_path): os.makedirs(output_path) @@ -358,27 +336,23 @@ def expanded_spec(self): expanded_filepath = os.path.join(self.info, expanded_name) # expand provenance spec filename - if contains_token(self.original_spec.name) or contains_shell_ref( - self.original_spec.name - ): + if contains_token(self.original_spec.name): name = f"{result.description['name'].replace(' ', '_')}_{self.timestamp}" - name = expand_line(name, {}, env_vars=True) if "/" in name: raise ValueError( f"Expanded value '{name}' for field 'name' in section 'description' is not a valid filename." ) expanded_workspace = os.path.join(self.output_path, name) - if result.merlin["samples"]: - sample_file = result.merlin["samples"]["file"] - if sample_file.startswith(self.workspace): - new_samples_file = sample_file.replace( - self.workspace, expanded_workspace - ) - result.merlin["samples"]["generate"]["cmd"] = result.merlin[ - "samples" - ]["generate"]["cmd"].replace(self.workspace, expanded_workspace) - result.merlin["samples"]["file"] = new_samples_file + sample_file = result.merlin["samples"]["file"] + if sample_file.startswith(self.workspace): + new_samples_file = sample_file.replace( + self.workspace, expanded_workspace + ) + result.merlin["samples"]["generate"]["cmd"] = result.merlin["samples"][ + "generate" + ]["cmd"].replace(self.workspace, expanded_workspace) + result.merlin["samples"]["file"] = new_samples_file shutil.move(self.workspace, expanded_workspace) self.workspace = expanded_workspace @@ -390,21 +364,8 @@ def expanded_spec(self): result.dump(), MerlinStudy.get_user_vars(result) ) result = MerlinSpec.load_spec_from_string(new_spec_text) - result = expand_env_vars(result) - - # pgen - if self.pgen_file: - env = result.get_study_environment() - result.globals = self.load_pgen(self.pgen_file, self.pargs, env) - - # copy the --samplesfile (if any) into merlin_info - if self.samples_file: - shutil.copyfile( - self.samples_file, - os.path.join(self.info, os.path.basename(self.samples_file)), - ) - # write expanded spec for provenance + # write expanded spec for provanance with open(expanded_filepath, "w") as f: f.write(result.dump()) @@ -430,7 +391,7 @@ def expanded_spec(self): @cached_property def flux_command(self): """ - Returns the flux version. + Returns a the flux version """ flux_bin = "flux" if "flux_path" in self.expanded_spec.batch.keys(): @@ -477,36 +438,14 @@ def generate_samples(self): LOG.error(f"Could not generate samples:\n{e}") return - def load_pgen(self, filepath, pargs, env): - if filepath: - if pargs is None: - pargs = [] - kwargs = create_dictionary(pargs) - params = load_parameter_generator(filepath, env, kwargs) - result = {} - for k, v in params.labels.items(): - result[k] = {"values": None, "label": v} - for k, v in params.parameters.items(): - result[k]["values"] = v - return result - def load_dag(self): """ Generates a dag (a directed acyclic execution graph). Assigns it to `self.dag`. """ - # TODO move this logic to specification.py - for key in ["variables", "labels", "sources", "dependencies"]: - if key not in self.expanded_spec.environment: - continue - if self.expanded_spec.environment[key] is None: - self.expanded_spec.environment[key] = {} environment = self.expanded_spec.get_study_environment() steps = self.expanded_spec.get_study_steps() - # TODO move this logic to specification.py - if self.expanded_spec.globals is None: - self.expanded_spec.globals = {} parameters = self.expanded_spec.get_parameters() # Setup the study. diff --git a/merlin/utils.py b/merlin/utils.py index 597ccee05..a7fd8b65c 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/setup.py b/setup.py index 19fb65d3d..26e7d90d6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 6a4ef7f27..e4ded4b5e 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -6,7 +6,7 @@ # # LLNL-CODE-797170 # All rights reserved. -# This file is part of Merlin, Version: 1.7.3. +# This file is part of Merlin, Version: 1.6.2. # # For details, see https://github.com/LLNL/merlin. # @@ -156,10 +156,8 @@ def run_tests(args, tests): continue try: passed, info = run_single_test(test_name, test, test_label) - except BaseException as e: - print(e) + except BaseException: passed = False - info = None result = process_test_result(passed, info, args.verbose, args.exit) if result is None: @@ -400,7 +398,6 @@ def define_tests(): purge = "merlin purge" examples = "merlin/examples/workflows" demo = f"{examples}/feature_demo/feature_demo.yaml" - demo_pgen = f"{examples}/feature_demo/scripts/pgen.py" simple = f"{examples}/simple_chain/simple_chain.yaml" slurm = f"{examples}/slurm/slurm_test.yaml" slurm_restart = f"{examples}/slurm/slurm_par_restart.yaml" @@ -590,31 +587,6 @@ def define_tests(): [RegexCond("2 samples loaded."), ReturnCodeCond()], "local", ), - "local pgen feature_demo": ( - f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ - ProvenanceCond( - regex="\[0.3333333", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - ), - ProvenanceCond( - regex="\[0.5", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - negate=True, - ), - ReturnCodeCond(), - ], - "local", - ), - # "local provenance spec equality": ( - # f"{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1 ; rm -rf ./{OUTPUT_DIR}/simple_chain_* ; {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')", - # ReturnCodeCond(), - # "local", - # ), "distributed feature_demo": ( f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers", [ @@ -654,9 +626,7 @@ def setup_argparse(): parser.add_argument( "--verbose", action="store_true", help="Flag for more detailed output messages" ) - parser.add_argument( - "--local", action="store_true", default=None, help="Run only local tests" - ) + parser.add_argument("--local", action="store_true", help="Run only local tests") parser.add_argument( "--ids", action="store", diff --git a/tests/study/test_study.py b/tests/study/test_study.py index c073f2078..0e3dd3a7f 100644 --- a/tests/study/test_study.py +++ b/tests/study/test_study.py @@ -21,7 +21,6 @@ env: variables: OUTPUT_PATH: ./studies - PATH_VAR: $PATH labels: SHARED: $(SPECROOT)/../shared @@ -265,11 +264,8 @@ def test_expanded_spec(self): assert not TestMerlinStudy.file_contains_string( self.study.expanded_spec.path, "$(OUTPUT_PATH)" ) - assert TestMerlinStudy.file_contains_string( - self.study.expanded_spec.path, "$PATH" - ) assert not TestMerlinStudy.file_contains_string( - self.study.expanded_spec.path, "PATH_VAR: $PATH" + self.study.expanded_spec.path, "$PATH" ) def test_column_label_conflict(self):