Maestro 1.1.8+ update (#233)

* working on adding maestro's schema validation to merlin * updated schemas * adding merlin_schema to spec logic * added debug prints * added print * fixing vestigial Maestro calls * adjusted merlin_schemas * fixed json file * validation progress * added from_dict replacement logic * adjusted maestrowf version req * fixed string * schema improvements * merlinsection schema updates * merlin section error parsing appears to work * updated CHANGELOG * fixed merge error * Added walltime to json schema
LLNL · Jun 18, 2020 · 011d126 · 011d126
1 parent e913381
commit 011d126
Show file tree

Hide file tree

Showing 7 changed files with 601 additions and 54 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,10 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
-- The sample generation command now logs stdout, stderr, and cmd.sh to merlin_info.
+- Integrated Maestro validation into merlin.
+- The sample `generate.cmd` now logs stdout, stderr, and `cmd.sh` to `merlin_info/`.
 
 ### Fixed
 - Corrected prior bugfix.
+- Unfroze Maestro version dependency.
 
 ## [1.6.1]
 

diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py
@@ -27,3 +27,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 ###############################################################################
+import os
+
+
+dirpath = os.path.dirname(os.path.abspath(__file__))
+SCHEMA_PATH = os.path.join(dirpath, "merlin_schemas.json")
diff --git a/merlin/spec/schemas/merlinsection.json b/merlin/spec/schemas/merlinsection.json
@@ -0,0 +1,144 @@
+{
+  "MERLIN": {
+    "type": "object",
+    "properties": {
+      "resources": {
+        "type": "object",
+        "properties": {
+          "task_server": {
+            "type": "string",
+            "minLength": 1
+          },
+          "overlap": {
+            "anyOf": [
+              {
+                "type": "boolean"
+              },
+              {
+                "type": "string",
+                "pattern": "^\\$\\(\\w+\\)$"
+              }
+            ]
+          },
+          "workers": {
+            "type": "object",
+            "additionalProperties": true,
+            "patternProperties": {
+              ".*": {
+                "type": "object",
+                "properties": {
+                  "steps": {
+                    "anyOf": [
+                      {
+                        "type": "array",
+                        "items": {
+                          "type": "string",
+                          "minLength": 1
+                        }
+                      },
+                      {
+                        "type": "string"
+                      }
+                    ]
+                  },
+                  "nodes": {
+                    "anyOf": [
+                      {
+                        "type": "integer",
+                        "minimum": 1
+                      },
+                      {
+                        "type": "string",
+                        "pattern": "^\\$\\(\\w+\\)$"
+                      },
+                      {
+                        "type": "null"
+                      }
+                    ]
+                  },
+                  "batch": {
+                    "anyOf": [
+                      {
+                        "type": "object"
+                      },
+                      {
+                        "type": "null"
+                      }
+                    ]
+                  },
+                  "args": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "machines": {
+                    "type": "array",
+                    "uniqueItems": true,
+                    "items": {
+                      "type": "string",
+                      "minLength": 1
+                    }
+                  }
+                },
+                "additionalProperties": false,
+                "required": [
+                  "steps"
+                ]
+              }
+            }
+          }
+        },
+        "additionalProperties": false
+      },
+      "samples": {
+        "anyOf": [
+          {
+            "type": "object",
+            "properties": {
+              "generate": {
+                "type": "object",
+                "properties": {
+                  "cmd": {
+                    "type": "string",
+                    "minLength": 1
+                  }
+                }
+              },
+              "level_max_dirs": {
+                "anyOf": [
+                  {
+                    "type": "integer",
+                    "minimum": 1
+                  },
+                  {
+                    "type": "string",
+                    "pattern": "^\\$\\(\\w+\\)$"
+                  }
+                ]
+              },
+              "file": {
+                "type": "string",
+                "minLength": 1
+              },
+              "column_labels": {
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "minLength": 1
+                }
+              }
+            },
+            "additionalProperties": false
+          },
+          {
+            "type": "null"
+          }
+        ]
+      }
+    },
+    "additionalProperties": false,
+    "required": [
+      "samples",
+      "resources"
+    ]
+  }
+}