Improvements to regression test python scripts

.github/workflows/automated-dev-tests.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
       - name: Setup workspace
         run: cmake -E make_directory ${{runner.workspace}}/openfast/build
       - name: Configure build
@@ -90,7 +90,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
       - name: Setup workspace
         run: cmake -E make_directory ${{runner.workspace}}/openfast/build
       - name: Configure build
@@ -132,7 +132,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev  # gcovr
       - name: Setup workspace
@@ -182,7 +182,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Build OpenFAST C-Interfaces
@@ -218,7 +218,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Build OpenFAST glue-code
@@ -249,7 +249,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Build FAST.Farm
@@ -287,7 +287,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Run AeroDyn tests
@@ -332,7 +332,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -387,7 +387,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Run Interface / API tests
@@ -424,7 +424,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -471,7 +471,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -515,7 +515,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -559,7 +559,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -603,7 +603,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -647,7 +647,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -691,7 +691,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests
@@ -735,7 +735,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install numpy Bokeh
+          pip install numpy "Bokeh>=2.4"
           sudo apt-get update -y
           sudo apt-get install -y libhdf5-dev libopenmpi-dev libyaml-cpp-dev
       - name: Configure Tests

reg_tests/executeAerodynRegressionCase.py

@@ -109,7 +109,7 @@
     caseInputFile = os.path.join(testBuildDirectory, "ad_driver.dvr")
     returnCode = openfastDrivers.runAerodynDriverCase(caseInputFile, executable, verbose=verbose)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 # For multiple turbines, test turbine 2, for combined cases, test case 4 

reg_tests/executeBeamdynRegressionCase.py

@@ -100,7 +100,7 @@
     caseInputFile = os.path.join(testBuildDirectory, "bd_driver.inp")
     returnCode = openfastDrivers.runBeamdynDriverCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, "bd_driver.out")

reg_tests/executeFASTFarmRegressionCase.py

@@ -123,7 +123,7 @@ def ignoreBaselineItems(directory, contents):
     caseInputFile = os.path.join(testBuildDirectory, caseName + ".fstf")
     returnCode = openfastDrivers.runOpenfastCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, caseName + ".out")

reg_tests/executeHydrodynPyRegressionCase.py

@@ -107,7 +107,7 @@
     caseInputFile = os.path.join(testBuildDirectory, "hydrodyn_driver.py")
     returnCode = openfastDrivers.runHydrodynDriverCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, "hd_py.out")

reg_tests/executeHydrodynRegressionCase.py

@@ -49,9 +49,9 @@
 parser.add_argument("buildDirectory", metavar="path/to/openfast_repo/build", type=str, nargs=1, help="The path to the OpenFAST repository build directory.")
 parser.add_argument("rtol", metavar="Relative-Tolerance", type=float, nargs=1, help="Relative tolerance to allow the solution to deviate; expressed as order of magnitudes less than baseline.")
 parser.add_argument("atol", metavar="Absolute-Tolerance", type=float, nargs=1, help="Absolute tolerance to allow small values to pass; expressed as order of magnitudes less than baseline.")
-parser.add_argument("-p", "-plot", dest="plot", default=False, metavar="Plotting-Flag", type=bool, nargs="?", help="bool to include matplotlib plots in failed cases")
-parser.add_argument("-n", "-no-exec", dest="noExec", default=False, metavar="No-Execution", type=bool, nargs="?", help="bool to prevent execution of the test cases")
-parser.add_argument("-v", "-verbose", dest="verbose", default=False, metavar="Verbose-Flag", type=bool, nargs="?", help="bool to include verbose system output")
+parser.add_argument("-p", "-plot", dest="plot", default=False, const=True, metavar="Plotting-Flag", type=bool, nargs="?", help="bool to include matplotlib plots in failed cases")
+parser.add_argument("-n", "-no-exec", dest="noExec", default=False, const=True, metavar="No-Execution", type=bool, nargs="?", help="bool to prevent execution of the test cases")
+parser.add_argument("-v", "-verbose", dest="verbose", default=False, const=True, metavar="Verbose-Flag", type=bool, nargs="?", help="bool to include verbose system output")
 
 args = parser.parse_args()
 
@@ -98,13 +98,19 @@
     for file in glob.glob(os.path.join(inputsDirectory,"*dat")):
         filename = file.split(os.path.sep)[-1]
         shutil.copy(os.path.join(inputsDirectory,filename), os.path.join(testBuildDirectory,filename))
+
+dirToCopy = os.path.join("glue-codes","openfast","5MW_Baseline","HydroData")
+buildDirectoryGlue = os.path.join(buildDirectory,os.pardir,os.pardir,dirToCopy)
+if not os.path.isdir(buildDirectoryGlue):
+    src = os.path.join(rtest,dirToCopy)
+    shutil.copytree(src, buildDirectoryGlue)
 
 ### Run HydroDyn on the test case
 if not noExec:
     caseInputFile = os.path.join(testBuildDirectory, "hd_driver.inp")
     returnCode = openfastDrivers.runHydrodynDriverCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, "driver.HD.out")

reg_tests/executeInflowwindPyRegressionCase.py

@@ -104,8 +104,8 @@
     caseInputFile = os.path.join(testBuildDirectory, "inflowWind_testDriver.py")
     returnCode = openfastDrivers.runInflowwindDriverCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
-
+        sys.exit(returnCode*10)
+    
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, "Points.Velocity.dat")
 baselineOutFile = os.path.join(targetOutputDirectory, "Points.Velocity.dat")

reg_tests/executeInflowwindRegressionCase.py

@@ -101,7 +101,7 @@
     caseInputFile = os.path.join(testBuildDirectory, "ifw_driver.inp")
     returnCode = openfastDrivers.runInflowwindDriverCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, "Points.Velocity.dat")

reg_tests/executeOpenfastAeroAcousticRegressionCase.py

@@ -109,8 +109,8 @@ def ignoreBaselineItems(directory, contents):
     caseInputFile = os.path.join(testBuildDirectory, caseName + ".fst")
     returnCode = openfastDrivers.runOpenfastCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
-
+        sys.exit(returnCode*10)
+    
 ### Build the filesystem navigation variables for running the regression test
 # testing on file 2.  Gives each observer and sweep of frequency ranges
 localOutFile = os.path.join(testBuildDirectory, caseName + "_2.out")

reg_tests/executeOpenfastCppRegressionCase.py

@@ -116,7 +116,7 @@ def ignoreBaselineItems(directory, contents):
     caseInputFile = os.path.abspath("cDriver.yaml")
     returnCode = openfastDrivers.runOpenfastCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
     os.chdir(cwd)
 
 ### Build the filesystem navigation variables for running the regression test

reg_tests/executeOpenfastLinearRegressionCase.py

@@ -145,7 +145,7 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
     caseInputFile = os.path.join(testBuildDirectory, caseName + ".fst")
     returnCode = openfastDrivers.runOpenfastCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Get a all the .lin files in the baseline directory
 baselineOutFiles = [f for f in os.listdir(targetOutputDirectory) if '.lin' in f]

reg_tests/executeOpenfastRegressionCase.py

@@ -130,7 +130,7 @@ def ignoreBaselineItems(directory, contents):
     caseInputFile = os.path.join(testBuildDirectory, caseName + ".fst")
     returnCode = openfastDrivers.runOpenfastCase(caseInputFile, executable)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, caseName + ".outb")

reg_tests/executePythonRegressionCase.py

@@ -146,6 +146,7 @@ def ignoreBaselineItems(directory, contents):
     output_channel_names = openfastlib.output_channel_names
 
 ### Build the filesystem navigation variables for running the regression test
+localOutFile = os.path.join(testBuildDirectory, caseName + ".outb")
 baselineOutFile = os.path.join(targetOutputDirectory, caseName + ".outb")
 rtl.validateFileOrExit(baselineOutFile)
 

reg_tests/executeSubdynRegressionCase.py

@@ -100,7 +100,7 @@
     # run driver
     returnCode = openfastDrivers.runSubdynDriverCase(caseInputFile, executable, verbose=verbose)
     if returnCode != 0:
-        rtl.exitWithError("")
+        sys.exit(returnCode*10)
 
 ### Build the filesystem navigation variables for running the regression test
 localOutFile = os.path.join(testBuildDirectory, caseName+".SD.out")

reg_tests/lib/errorPlotting.py

@@ -208,23 +208,30 @@ def finalizePlotDirectory(test_solution, plot_list, case):
 
         for i, plot in enumerate(plot_list):
             _path = os.path.join(plot_path, plot + '_div.txt')
-            with open(_path, 'r') as f:
-                div = f.read().strip().join(('      ', '\n'))
-            html = ''.join((html, div))
+            try:
+                with open(_path, 'r') as f:
+                    div = f.read().strip().join(('      ', '\n'))
+                html = ''.join((html, div))
+            except FileNotFoundError:
+                print("The file for ", plot ," does not exist.")
 
         html = ''.join((html, '    </div>' + '\n'))
         html = ''.join((html, '  </div>' + '\n'))
         html = ''.join((html, '</body>' + '\n'))
         html = ''.join((html, _htmlTail()))
 
+    script = "" # initialize in case plot_list is empty
     for i, plot in enumerate(plot_list):
         _path = os.path.join(plot_path, f'{plot}_script.txt')
-        with open(_path, "r") as f:
-            _s = f.read()
-        if i == 0:
-            script = _s
-        else:
-            script = ''.join((script, _s))
+        try:
+            with open(_path, "r") as f:
+                _s = f.read()
+            if i == 0:
+                script = _s
+            else:
+                script = ''.join((script, _s))
+        except FileNotFoundError:
+            msg = "The file does not exist."
 
     shutil.rmtree(plot_path, ignore_errors=True)
 
@@ -243,8 +250,8 @@ def exportResultsSummary(path, results):
         html.write('  <div class="container">' + '\n')
 
         # Test Case - Pass/Fail - Max Relative Norm            
-        data = [('<a href="{0}/{0}.html">{0}</a>'.format(r[0]), r[1]) for i,r in enumerate(results)]
-        table = _tableHead(['Test Case', 'Pass/Fail'])
+        data = [('<a href="{0}/{0}.html">{0}</a>'.format(r[0]), r[1],r[2],'<a href="{0}/{0}.log">{0}.log</a>'.format(r[0])) for i,r in enumerate(results)]
+        table = _tableHead(['Test Case', 'Pass/Fail', 'Completion Code', 'Screen Output'])
         body = '      <tbody>' + '\n'
         for i, d in enumerate(data):
             body += '        <tr>' + '\n'
@@ -256,7 +263,14 @@ def exportResultsSummary(path, results):
                 body += ('          <td class="cell-warning">' + fmt + '</td>').format(d[1]) + '\n'
             else:
                 body += ('          <td>' + fmt + '</td>').format(d[1]) + '\n'
+
+            if d[2] != 0:
+                body += ('          <td class="cell-warning">{}</td>').format(d[2]) + '\n'
+            else:
+                body += ('          <td>{}</td>').format(d[2]) + '\n'
 
+            body += '          <td>{0:s}</td>'.format(d[3]) + '\n'
+
             body += '        </tr>' + '\n'
         body += '      </tbody>' + '\n'
         table += body

reg_tests/lib/openfastDrivers.py

@@ -41,11 +41,12 @@ def _runGenericCase(inputFile, executable, verbose=False, log=True):
     rtl.validateFileOrExit(inputFile)
     rtl.validateExeOrExit(executable)
 
-    casebase = os.path.sep.join(inputFile.split(os.path.sep)[-1].split('.')[:-1])
-    caseparent = os.path.sep.join(inputFile.split(os.path.sep)[:-1])
+#bjj, why wouldn't we check if log is true instead of verbose being false to generate a log file???
     if verbose:
         logFile = None
     else:
+        caseparent = os.path.sep.join(inputFile.split(os.path.sep)[:-1])
+        casebase = caseparent.split(os.path.sep)[-1]  # assumes that the directory structure name is the name of the .log file. (for consistent driver + glue-code names)
         logFile = caseparent + os.path.sep + casebase + '.log'
 
     returnCode = _runCase(executable, inputFile, logFile, stdout)

reg_tests/lib/pass_fail.py

@@ -95,9 +95,17 @@ def calculate_max_norm(testData, baselineData):
     return maxnorm(abs(testData - baselineData))
 
 def calculateNorms(test_data, baseline_data):
-    relative_norm = calculate_max_norm_over_range(test_data, baseline_data)
-    max_norm = calculate_max_norm(test_data, baseline_data)
-    relative_l2_norm = calculate_relative_norm(test_data, baseline_data)
+    if test_data.size != baseline_data.size:
+       # print("Calculate Norms size(testdata)={}".format(test_data.size)) 
+       # print("Calculate Norms size(baseline)={}".format(baseline_data.size))
+       relative_norm = np.nan * calculate_max_norm_over_range(test_data, test_data)
+       max_norm = relative_norm
+       relative_l2_norm = relative_norm
+    else:
+       relative_norm = calculate_max_norm_over_range(test_data, baseline_data)
+       max_norm = calculate_max_norm(test_data, baseline_data)
+       relative_l2_norm = calculate_relative_norm(test_data, baseline_data)
+
     results = np.stack(
         (
             relative_norm,