YAMLSpecification Testing (#317)

* initial testing on loading spec

* black settings added

* reformatted by black

* added test for validation errors

* added test to check for missing key in step

* updated schema to add check for non-null variables

* added check for study steps

* added test for multiple dependencies of the same name

* changed yaml spec testing for easier reading

* added test of global parameters

* fixed test execution to verify error thrown

* added output_path test

* formatting fix

* added tests for steps generation and params generation

* added tests for get_study_env

* initial testing on loading spec

* black settings added

* reformatted by black

* added test for validation errors

* added test to check for missing key in step

* updated schema to add check for non-null variables

* added check for study steps

* added test for multiple dependencies of the same name

* changed yaml spec testing for easier reading

* added test of global parameters

* fixed test execution to verify error thrown

* added output_path test

* formatting fix

* added tests for steps generation and params generation

* added tests for get_study_env

* updated to add testing for description setter

* added test for spec.name setter

* pulled spec_path code into a pytest fixture

* added pytest and coverage config

* added pytest-cov as a tox requirement

* added report.xml to gitignore

* changed line length for black

* added coverage and junit reports to tox

* Added coverage appending for test runs

.gitignore

@@ -22,6 +22,7 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+report.xml
 *.cover
 .hypothesis/
 .pytest_cache/

maestrowf/specification/schemas/yamlspecification.json

@@ -82,7 +82,17 @@
   "ENV": {
     "type": "object",
     "properties": {
-      "variables": {"type": "object"},
+      "variables": {
+        "type": "object",
+        "patternProperties": {
+          "^.*": {
+            "anyOf": [
+                {"type": "string", "minLength": 1},
+                {"type": "number"}
+            ]
+          }
+        }
+      },
       "labels": {"type": "object"},
       "sources": {"type": "array"},
       "dependencies": {

pyproject.toml

@@ -71,3 +71,6 @@ build-backend = "poetry.masonry.api"
 [tool.poetry.urls]
 "Bug Tracker" = "https://github.com/LLNL/maestrowf/issues"
 "Discussions" = "https://github.com/LLNL/maestrowf/discussions"
+
+[tool.black]
+line-length = 79

pytest.ini

@@ -1,2 +1,3 @@
 [pytest]
 addopts = -p no:warnings --cov-config=.coveragerc  --cov-report=term-missing --cov=.
+junit_family = xunit2

tests/conftest.py

@@ -0,0 +1,11 @@
+import pytest
+import os
+
+
+@pytest.fixture
+def spec_path():
+    def load_spec(file_name):
+        dirpath = os.path.dirname(os.path.abspath(__file__))
+        return os.path.join(dirpath, "specification", "test_specs", file_name)
+
+    return load_spec

tests/specification/test_specs/duplicate_dependency.yml

@@ -0,0 +1,24 @@
+description:
+  name: hello_world
+  description: A simple 'Hello World' study.
+
+env:
+  variables:
+    OUTPUT_PATH: ./sample_output/hello_world
+  dependencies:
+    git:
+      - name: LULESH
+        path: $(OUTPUT_PATH)
+        url: https://github.com/LLNL/LULESH.git
+        tag: 2.0.3
+      - name: LULESH
+        path: $(OUTPUT_PATH)
+        url: https://github.com/LLNL/LULESH.git
+        tag: 2.0.3
+
+study:
+  - name: hello_world
+    description: Say hello to the world!
+    run:
+      cmd: |
+        echo "Hello, World!" > hello_world.txt

tests/specification/test_specs/empty_output_path.yml

@@ -0,0 +1,10 @@
+description:
+  name: hello_world
+  description: A simple 'Hello World' study.
+
+study:
+  - name: hello_world
+    description: Say hello to the world!
+    run:
+      cmd: |
+        echo "Hello, World!" > hello_world.txt

tests/specification/test_specs/empty_variables.yml

@@ -0,0 +1,14 @@
+description:
+  name: hello_world
+  description: A simple Hello World study.
+
+env:
+  variables:
+    OUTPUT_PATH:
+
+study:
+  - name: hello_world
+    description: Say hello to the world!
+    run:
+      cmd: |
+           echo "Hello, World!" > hello_world.txt

tests/specification/test_specs/error_parameterized.yml

@@ -0,0 +1,34 @@
+description:
+    name: hello_bye_world
+    description: A study that says hello and bye to multiple people.
+
+env:
+    variables:
+        OUTPUT_PATH: ./sample_output/hello_world
+    labels:
+        OUT_FORMAT: $(GREETING)_$(NAME).txt
+
+study:
+    - name: hello_world
+      description: Say hello to someone!
+      run:
+          cmd: |
+            echo "$(GREETING), $(NAME)!" > $(OUT_FORMAT)
+
+    - name: bye_world
+      description: Say bye to someone!
+      run:
+          cmd: |
+            echo "Bye, World!" > bye.txt
+          depends: [hello_world]
+
+global.parameters:
+    NAME:
+        values: [Pam, Jim, Michael, Dwight]
+        label: NAME.%%
+    GREETING:
+        values: [Hello, Ciao, Hey, Hi]
+        label: GREETING.%%
+    GREETING2:
+      values: [ Hello, Ciao, Hey, Hi ]
+      label: [GREETING.%%]

tests/specification/test_specs/extra_study_params.yml

@@ -0,0 +1,15 @@
+description:
+  name: hello_world
+  description: A simple 'Hello World' study.
+
+env:
+  variables:
+    OUTPUT_PATH: ./sample_output/hello_world
+
+study:
+  - name: hello_world
+    description: yikes
+    run:
+      bad: this will fail
+      cmd: |
+        echo "Hello, World!" > hello_world.txt

tests/specification/test_specs/hello_bye_parameterized.yml

@@ -0,0 +1,31 @@
+description:
+    name: hello_bye_world
+    description: A study that says hello and bye to multiple people.
+
+env:
+    variables:
+        OUTPUT_PATH: ./sample_output/hello_world
+    labels:
+        OUT_FORMAT: $(GREETING)_$(NAME).txt
+
+study:
+    - name: hello_world
+      description: Say hello to someone!
+      run:
+          cmd: |
+            echo "$(GREETING), $(NAME)!" > $(OUT_FORMAT)
+
+    - name: bye_world
+      description: Say bye to someone!
+      run:
+          cmd: |
+            echo "Bye, World!" > bye.txt
+          depends: [hello_world]
+
+global.parameters:
+    NAME:
+        values: [Pam, Jim, Michael, Dwight]
+        label: NAME.%%
+    GREETING:
+        values: [Hello, Ciao, Hey, Hi]
+        label: GREETING.%%

tests/specification/test_specs/hello_world.yml

@@ -0,0 +1,14 @@
+description:
+    name: hello_world
+    description: A simple 'Hello World' study.
+
+env:
+    variables:
+        OUTPUT_PATH: ./sample_output/hello_world
+
+study:
+    - name: hello_world
+      description: Say hello to the world!
+      run:
+          cmd: |
+            echo "Hello, World!" > hello_world.txt

tests/specification/test_specs/lulesh_sample1_unix.yml

@@ -0,0 +1,90 @@
+description:
+    name: lulesh_sample1
+    description: A sample LULESH study that downloads, builds, and runs a parameter study of varying problem sizes and iterations.
+
+env:
+    sources:
+      - git
+    variables:
+        # DEFAULTS FOR MONTECARLO PGEN EXAMPLE
+        SMIN:        10
+        SMAX:        30
+        TRIALS:      50
+        ITER:        100
+
+        OUTPUT_PATH: ./sample_output/lulesh
+
+    labels:
+        outfile: $(SIZE.label).$(ITERATIONS.label).log
+
+    dependencies:
+      git:
+        - name: LULESH
+          path: $(OUTPUT_PATH)
+          url: https://github.com/LLNL/LULESH.git
+          tag: 2.0.3
+      paths:
+        - name: TEST
+          path: /test/
+
+study:
+    - name: make-lulesh
+      description: Build the serial version of LULESH.
+      run:
+          cmd: |
+            cd $(LULESH)
+            sed -i 's/^CXX = $(MPICXX)/CXX = $(SERCXX)/' ./Makefile
+            sed -i 's/^CXXFLAGS = -g -O3 -fopenmp/#CXXFLAGS = -g -O3 -fopenmp/' ./Makefile
+            sed -i 's/^#LDFLAGS = -g -O3/LDFLAGS = -g -O3/' ./Makefile
+            sed -i 's/^LDFLAGS = -g -O3 -fopenmp/#LDFLAGS = -g -O3 -fopenmp/' ./Makefile
+            sed -i 's/^#CXXFLAGS = -g -O3 -I/CXXFLAGS = -g -O3 -I/' ./Makefile
+            make clean
+            make
+          depends: []
+
+    - name: run-lulesh
+      description: Run LULESH.
+      run:
+          cmd: |
+            $(LULESH)/lulesh2.0 -s $(SIZE) -i $(ITERATIONS) -p > $(outfile)
+          depends: [make-lulesh]
+
+    - name: post-process-lulesh
+      description: Post process all LULESH results.
+      run:
+          cmd: |
+            echo "Unparameterized step with Parameter Independent dependencies." >> out.log
+            echo $(run-lulesh.workspace) > out.log
+            ls $(run-lulesh.workspace) > ls.log
+          depends: [run-lulesh_*]
+
+    - name: post-process-lulesh-trials
+      description: Post process all LULESH results.
+      run:
+          cmd: |
+            echo "Parameterized step that has Parameter Independent dependencies" >> out.log
+            echo "TRIAL = $(TRIAL)" >> out.log
+            echo $(run-lulesh.workspace) >> out.log
+            ls $(run-lulesh.workspace) >> out.log
+          depends: [run-lulesh_*]
+
+    - name: post-process-lulesh-size
+      description: Post process all LULESH results.
+      run:
+          cmd: |
+            echo "Parameterized step that has Parameter Independent dependencies" >> out.log
+            echo "SIZE = $(SIZE)" >> out.log
+            echo $(run-lulesh.workspace) >> out.log
+            ls $(run-lulesh.workspace) | grep $(SIZE.label) >> out.log
+          depends: [run-lulesh_*]
+
+global.parameters:
+    TRIAL:
+        values  : [1, 2, 3, 4, 5, 6, 7, 8, 9]
+        label   : TRIAL.%%
+    SIZE:
+        values  : [10, 10, 10, 20, 20, 20, 30, 30, 30]
+        label   : SIZE.%%
+    ITERATIONS:
+        values  : [10, 20, 30, 10, 20, 30, 10, 20, 30]
+        label   : ITER.%%

tests/specification/test_specs/missing_step.yml

@@ -0,0 +1,10 @@
+description:
+  name: hello_world
+  description: A simple 'Hello World' study.
+
+env:
+  variables:
+    OUTPUT_PATH: ./sample_output/hello_world
+
+
+study:

tests/specification/test_specs/missing_step_desc.yml

@@ -0,0 +1,14 @@
+description:
+  name: hello_world
+  description: A simple 'Hello World' study.
+
+env:
+  variables:
+    OUTPUT_PATH: ./sample_output/hello_world
+
+
+study:
+  - name: hello_world
+    run:
+      cmd: |
+        echo "Hello, World!" > hello_world.txt