Remove use of travis-specific make targets for testcase unit tests

.travis.yml

@@ -74,22 +74,22 @@ jobs:
     script: cd testing && make build_jm_image && make test_twozone_apartment_hydronic
   - python: 3.9
     install: pip install --upgrade pip && pip install pandas==1.2.5 numpy==1.20.2 matplotlib==3.3.4 requests==2.25.1
-    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_office_simple_air_travis ARG="-s test_peak_heat_day,test_typical_heat_day"
+    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_office_simple_air ARG="-s test_peak_heat_day,test_typical_heat_day"
   - python: 3.9
     install: pip install --upgrade pip && pip install pandas==1.2.5 numpy==1.20.2 matplotlib==3.3.4 requests==2.25.1
-    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_office_simple_air_travis ARG="-s test_peak_cool_day,test_typical_cool_day,test_mix_day"
+    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_office_simple_air ARG="-s test_peak_cool_day,test_typical_cool_day,test_mix_day"
   - python: 3.9
     install: pip install --upgrade pip && pip install pandas==1.2.5 numpy==1.20.2 matplotlib==3.3.4 requests==2.25.1
-    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_office_simple_air_travis ARG="-s API"
+    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_office_simple_air ARG="-s API"
   - python: 3.9
     install: pip install --upgrade pip && pip install pandas==1.2.5 numpy==1.20.2 matplotlib==3.3.4 requests==2.25.1
-    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_residential_hydronic_travis ARG="-s test_peak_heat_day,test_shoulder"
+    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_residential_hydronic ARG="-s test_peak_heat_day,test_shoulder"
   - python: 3.9
     install: pip install --upgrade pip && pip install pandas==1.2.5 numpy==1.20.2 matplotlib==3.3.4 requests==2.25.1
-    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_residential_hydronic_travis ARG="-s test_typical_heat_day,test_summer"
+    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_residential_hydronic ARG="-s test_typical_heat_day,test_summer"
   - python: 3.9
     install: pip install --upgrade pip && pip install pandas==1.2.5 numpy==1.20.2 matplotlib==3.3.4 requests==2.25.1
-    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_residential_hydronic_travis ARG="-s API"
+    script: cd testing && make build_jm_image && travis_wait 90 make test_multizone_residential_hydronic ARG="-s API"
   - python: 2.7
     install: pip install --upgrade pip && pip install pandas==0.24.2 numpy==1.16.6 matplotlib==2.1.1 requests==2.18.4
     script: cd testing && make test_python2

releasenotes.md

@@ -15,6 +15,8 @@ Released on xx/xx/xxxx.
 - Add Git LFS configuration in the ``testing/Dockerfile`` image used in tests and compilation. This is for [#613](https://github.com/ibpsa/project1-boptest/issues/613).
 - Correct typo in documentation for ``multizone_office_simple_air``, cooling setback temperature changed from 12 to 30. This is for [#605](https://github.com/ibpsa/project1-boptest/issues/605).
 - Modify unit tests for test case scenarios to only simulate two days after warmup instead of the whole two-week scenario. This is for [#576](https://github.com/ibpsa/project1-boptest/issues/576).
+- Fix unit tests for possible false passes in certain test cases. This is for [#620](https://github.com/ibpsa/project1-boptest/issues/620).
+
 
 ## BOPTEST v0.5.0
 

testing/makefile

@@ -87,17 +87,7 @@ generate_testcase_data:
 # Tests
 ###############################################################################
 
-# Generic test for all testcases except testcase1 and testcase2 to run on travis
-test_%_travis:
-# Compile testcase model
-	make compile_testcase_model TESTCASE=$*
-# Build and deploy testcase image
-	cd .. && TESTCASE=$* docker-compose up -d
-	python sleep10.py
-# Run testcase tests
-	cd .. && python testing/test_$*.py $(ARG)
-
-# Generic test for all testcases except testcase1 and testcase2 to run locally with cleanup
+# Generic test for all testcases except testcase1 and testcase2
 test_%:
 # Compile testcase model
 	make compile_testcase_model TESTCASE=$*

testing/references/multizone_office_simple_air/kpis_mix_day_constant.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.024591006165327153
-time_rat,0.0018162296215693157
+cost_tot,0.0023115545795407526
 emis_tot,0.008385533102376561
+ener_tot,0.024591006165327153
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.0046098221232898275
-tdis_tot,0.6820425793461586
 pgas_tot,
-cost_tot,0.0023115545795407526
-pdih_tot,
+tdis_tot,0.6820425793461586
+time_rat,4.643367396460639e-08

testing/references/multizone_office_simple_air/kpis_mix_day_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.024591006165327153
-time_rat,0.0018162296215693157
+cost_tot,0.002220768604807324
 emis_tot,0.008385533102376561
+ener_tot,0.024591006165327153
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.0046098221232898275
-tdis_tot,0.6820425793461586
 pgas_tot,
-cost_tot,0.002220768604807324
-pdih_tot,
+tdis_tot,0.6820425793461586
+time_rat,4.643367396460639e-08

testing/references/multizone_office_simple_air/kpis_mix_day_highly_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.024591006165327153
-time_rat,0.0018162296215693157
+cost_tot,0.0019259669723876174
 emis_tot,0.008385533102376561
+ener_tot,0.024591006165327153
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.0046098221232898275
-tdis_tot,0.6820425793461586
 pgas_tot,
-cost_tot,0.0019259669723876174
-pdih_tot,
+tdis_tot,0.6820425793461586
+time_rat,4.643367396460639e-08

testing/references/multizone_office_simple_air/kpis_peak_cool_day_constant.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.23474213243308178
-time_rat,0.0009058127800623575
+cost_tot,0.02206576044870969
 emis_tot,0.08004706715968089
+ener_tot,0.23474213243308178
 idis_tot,13.730462389034788
+pdih_tot,
 pele_tot,0.01394586591124034
-tdis_tot,0.7996165371652224
 pgas_tot,
-cost_tot,0.02206576044870969
-pdih_tot,
+tdis_tot,0.7996165371652224
+time_rat,0.0009528091646454953

testing/references/multizone_office_simple_air/kpis_peak_cool_day_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.23474213243308178
-time_rat,0.0009058127800623575
+cost_tot,0.026538994084054388
 emis_tot,0.08004706715968089
+ener_tot,0.23474213243308178
 idis_tot,13.730462389034788
+pdih_tot,
 pele_tot,0.01394586591124034
-tdis_tot,0.7996165371652224
 pgas_tot,
-cost_tot,0.026538994084054388
-pdih_tot,
+tdis_tot,0.7996165371652224
+time_rat,0.0009528091646454953

testing/references/multizone_office_simple_air/kpis_peak_cool_day_highly_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.23474213243308178
-time_rat,0.0009058127800623575
+cost_tot,0.016933297405124552
 emis_tot,0.08004706715968089
+ener_tot,0.23474213243308178
 idis_tot,13.730462389034788
+pdih_tot,
 pele_tot,0.01394586591124034
-tdis_tot,0.7996165371652224
 pgas_tot,
-cost_tot,0.016933297405124552
-pdih_tot,
+tdis_tot,0.7996165371652224
+time_rat,0.0009528091646454953

testing/references/multizone_office_simple_air/kpis_peak_heat_day_constant.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.13934001725214568
-time_rat,0.0010747482199911717
+cost_tot,0.013097961621701694
 emis_tot,0.04751494588298167
+ener_tot,0.13934001725214568
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.021374577846507703
-tdis_tot,1.0527764845675012
 pgas_tot,
-cost_tot,0.013097961621701694
-pdih_tot,
+tdis_tot,1.0527764845675012
+time_rat,4.278288947211371e-08

testing/references/multizone_office_simple_air/kpis_peak_heat_day_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.13934001725214568
-time_rat,0.0010747482199911717
+cost_tot,0.012272815404686197
 emis_tot,0.04751494588298167
+ener_tot,0.13934001725214568
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.021374577846507703
-tdis_tot,1.0527764845675012
 pgas_tot,
-cost_tot,0.012272815404686197
-pdih_tot,
+tdis_tot,1.0527764845675012
+time_rat,4.278288947211371e-08

testing/references/multizone_office_simple_air/kpis_peak_heat_day_highly_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.13934001725214568
-time_rat,0.0010747482199911717
+cost_tot,0.009837388443659313
 emis_tot,0.04751494588298167
+ener_tot,0.13934001725214568
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.021374577846507703
-tdis_tot,1.0527764845675012
 pgas_tot,
-cost_tot,0.009837388443659313
-pdih_tot,
+tdis_tot,1.0527764845675012
+time_rat,4.278288947211371e-08

testing/references/multizone_office_simple_air/kpis_typical_cool_day_constant.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.028490034605776336
-time_rat,0.0006533037070874816
+cost_tot,0.002678063252942976
 emis_tot,0.009715101800569732
+ener_tot,0.028490034605776336
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.0022934447194918627
-tdis_tot,0.033545272379636136
 pgas_tot,
-cost_tot,0.002678063252942976
-pdih_tot,
+tdis_tot,0.033545272379636136
+time_rat,0.000690734818301819

testing/references/multizone_office_simple_air/kpis_typical_cool_day_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.028490034605776336
-time_rat,0.0006533037070874816
+cost_tot,0.0032575690703220243
 emis_tot,0.009715101800569732
+ener_tot,0.028490034605776336
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.0022934447194918627
-tdis_tot,0.033545272379636136
 pgas_tot,
-cost_tot,0.0032575690703220243
-pdih_tot,
+tdis_tot,0.033545272379636136
+time_rat,0.000690734818301819

testing/references/multizone_office_simple_air/kpis_typical_cool_day_highly_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.028490034605776336
-time_rat,0.0006533037070874816
+cost_tot,0.0018089984725032451
 emis_tot,0.009715101800569732
+ener_tot,0.028490034605776336
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.0022934447194918627
-tdis_tot,0.033545272379636136
 pgas_tot,
-cost_tot,0.0018089984725032451
-pdih_tot,
+tdis_tot,0.033545272379636136
+time_rat,0.000690734818301819

testing/references/multizone_office_simple_air/kpis_typical_heat_day_constant.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.20135378691066924
-time_rat,0.0013540542194688764
+cost_tot,0.01892725596960291
 emis_tot,0.06866164133653822
+ener_tot,0.20135378691066924
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.026283684748313458
-tdis_tot,1.2854827147383858
 pgas_tot,
-cost_tot,0.01892725596960291
-pdih_tot,
+tdis_tot,1.2854827147383858
+time_rat,0.0014178943026948857

testing/references/multizone_office_simple_air/kpis_typical_heat_day_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.20135378691066924
-time_rat,0.0013540542194688764
+cost_tot,0.018890807461897544
 emis_tot,0.06866164133653822
+ener_tot,0.20135378691066924
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.026283684748313458
-tdis_tot,1.2854827147383858
 pgas_tot,
-cost_tot,0.018890807461897544
-pdih_tot,
+tdis_tot,1.2854827147383858
+time_rat,0.0014178943026948857

testing/references/multizone_office_simple_air/kpis_typical_heat_day_highly_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.20135378691066924
-time_rat,0.0013540542194688764
+cost_tot,0.012915934356027807
 emis_tot,0.06866164133653822
+ener_tot,0.20135378691066924
 idis_tot,0.0
+pdih_tot,
 pele_tot,0.026283684748313458
-tdis_tot,1.2854827147383858
 pgas_tot,
-cost_tot,0.012915934356027807
-pdih_tot,
+tdis_tot,1.2854827147383858
+time_rat,0.0014178943026948857

testing/references/multizone_residential_hydronic/kpis_peak_heat_day_constant.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,1.0994019068555612
-time_rat,0.0038431207559726853
+cost_tot,0.10639064997723341
 emis_tot,0.19318817460065163
+ener_tot,1.0994019068555612
 idis_tot,1106.6064736299081
+pdih_tot,
 pele_tot,0.0017390231869758264
-tdis_tot,2.991582452944326
 pgas_tot,0.09592720495532532
-cost_tot,0.10639064997723341
-pdih_tot,
+tdis_tot,2.991582452944326
+time_rat,4.5078772085684314e-08

testing/references/multizone_residential_hydronic/kpis_peak_heat_day_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,1.0994019068555612
-time_rat,0.0038431207559726853
+cost_tot,0.10614042392472808
 emis_tot,0.19318817460065163
+ener_tot,1.0994019068555612
 idis_tot,1106.6064736299081
+pdih_tot,
 pele_tot,0.0017390231869758264
-tdis_tot,2.991582452944326
 pgas_tot,0.09592720495532532
-cost_tot,0.10614042392472808
-pdih_tot,
+tdis_tot,2.991582452944326
+time_rat,4.5078772085684314e-08

testing/references/multizone_residential_hydronic/kpis_peak_heat_day_highly_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,1.0994019068555612
-time_rat,0.0038431207559726853
+cost_tot,0.10413134235385925
 emis_tot,0.19318817460065163
+ener_tot,1.0994019068555612
 idis_tot,1106.6064736299081
+pdih_tot,
 pele_tot,0.0017390231869758264
-tdis_tot,2.991582452944326
 pgas_tot,0.09592720495532532
-cost_tot,0.10413134235385925
-pdih_tot,
+tdis_tot,2.991582452944326
+time_rat,4.5078772085684314e-08

testing/references/multizone_residential_hydronic/kpis_shoulder_constant.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.0024363681381090485
-time_rat,9.88688595868923e-06
+cost_tot,0.0005676737761794082
 emis_tot,0.00011450930249112526
+ener_tot,0.0024363681381090485
 idis_tot,1914.9068664587317
+pdih_tot,
 pele_tot,0.00032267245216700314
-tdis_tot,0.0
 pgas_tot,0.0
-cost_tot,0.0005676737761794082
-pdih_tot,
+tdis_tot,0.0
+time_rat,1.2715841057123961e-05

testing/references/multizone_residential_hydronic/kpis_shoulder_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.0024363681381090485
-time_rat,9.88688595868923e-06
+cost_tot,0.0005763367941799247
 emis_tot,0.00011450930249112526
+ener_tot,0.0024363681381090485
 idis_tot,1914.9068664587317
+pdih_tot,
 pele_tot,0.00032267245216700314
-tdis_tot,0.0
 pgas_tot,0.0
-cost_tot,0.0005763367941799247
-pdih_tot,
+tdis_tot,0.0
+time_rat,1.2715841057123961e-05

testing/references/multizone_residential_hydronic/kpis_shoulder_highly_dynamic.csv

@@ -1,10 +1,10 @@
 keys,value
-ener_tot,0.0024363681381090485
-time_rat,9.88688595868923e-06
+cost_tot,0.00040830259553863645
 emis_tot,0.00011450930249112526
+ener_tot,0.0024363681381090485
 idis_tot,1914.9068664587317
+pdih_tot,
 pele_tot,0.00032267245216700314
-tdis_tot,0.0
 pgas_tot,0.0
-cost_tot,0.00040830259553863645
-pdih_tot,
+tdis_tot,0.0
+time_rat,1.2715841057123961e-05