Skip to content

Commit 03fa12c

Browse files
tomstescotomstesco
authoredJan 20, 2021
Merge pull request #129 from ecobee/dev
fix output file names, add nullable dtype handling, upgrade to pandas 1.2.1
·
0.6.0-alpha0.3.3-alpha
2 parents f58698d + 2fa94fc commit 03fa12c

22 files changed

+281
-115
lines changed
 

‎.dockerignore

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
*
2+
!Dockerfile
3+
!src/
4+
!scripts/
5+
!setup.py
6+
!requirements.txt
7+
!requirements_unfixed.txt
8+
!pytest.ini

‎.env.template

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
PACKAGE_NAME=building-controls-simulator
2-
VERSION_TAG=0.3.2-alpha
2+
VERSION_TAG=0.3.3-alpha
3+
DOCKERHUB_REPOSITORY=tstesco
34
USER_NAME=bcs
45
DOCKER_IMAGE=${PACKAGE_NAME}
5-
LOCAL_PACKAGE_DIR=<where you cloned repo>
6+
LOCAL_PACKAGE_DIR=<where you cloned the repo>
67
DOCKER_HOME_DIR=/home/${USER_NAME}
78
DOCKER_LIB_DIR=${DOCKER_HOME_DIR}/lib
89
DOCKER_PACKAGE_DIR=${DOCKER_LIB_DIR}/${PACKAGE_NAME}
@@ -14,7 +15,7 @@ PACKAGE_DIR=/home/bcs/lib/${PACKAGE_NAME}
1415
PYENV_ROOT=${DOCKER_HOME_DIR}/pyenv
1516
ENERGYPLUS_INSTALL_DIR=${EXT_DIR}/EnergyPlus
1617
ENERGYPLUSTOFMUSCRIPT=${EXT_DIR}/EnergyPlusToFMU-3.0.0/Scripts/EnergyPlusToFMU.py
17-
GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json
18+
# GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json
1819
# DYD_GOOGLE_CLOUD_PROJECT=<GCP project for DYD access>
1920
# DYD_GCS_URI_BASE=<GCP DYD bucket filter>
2021
# DYD_METADATA_URI=<GCP DYD bucket URI to metadata file>

‎.gitignore

+5-1
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,15 @@ data/
2020
weather/
2121
idf/
2222
fmu/
23+
notebooks/
24+
notes/
2325
!test/idf/v8-9-0/AZ_Phoenix_gasfurnace_crawlspace_IECC_2018_cycles.idf
2426
!test/idf/v9-4-0/heatedbsmt_2story_2300sqft_gasfurnace_AC.idf
2527
!test/idf/v9-4-0/heatedbsmt_1story_2000sqft_gasfurnace_AC.idf
2628
!test/idf/v9-4-0/slab_1story_2000sqft_gasfurnace_AC.idf
2729
!test/fmu/fmu-models/deadband/deadband.fmu
30+
!requirements.txt
31+
!requirements_unfixed.txt
2832
__pycache__/
2933
.pytest_cache/
3034
.ipynb_checkpoints/
@@ -35,4 +39,4 @@ docs/source/generated/
3539
.test.env
3640
docker-compose.yml
3741
build/
38-
notebooks/test_*
42+
!notebooks/test_*

‎.test.env.template

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ ENERGYPLUS_INSTALL_VERSION=9-4-0
1010
EPLUS_DIR=${ENERGYPLUS_INSTALL_DIR}/EnergyPlus-${ENERGYPLUS_INSTALL_VERSION}
1111
EPLUS_IDD=${ENERGYPLUS_INSTALL_DIR}/EnergyPlus-${ENERGYPLUS_INSTALL_VERSION}/PreProcess/IDFVersionUpdater/V${ENERGYPLUS_INSTALL_VERSION}-Energy+.idd
1212
ENERGYPLUSTOFMUSCRIPT=${EXT_DIR}/EnergyPlusToFMU-3.0.0/Scripts/EnergyPlusToFMU.py
13-
GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json
13+
# GOOGLE_APPLICATION_CREDENTIALS=${DOCKER_HOME_DIR}/.config/gcloud/application_default_credentials.json
1414
# DYD_GOOGLE_CLOUD_PROJECT=<GCP project for DYD access>
1515
# DYD_GCS_URI_BASE=<GCP DYD bucket filter>
1616
# DYD_METADATA_URI=<GCP DYD bucket URI to metadata file>

‎Dockerfile

+8-12
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ RUN sudo apt-get update && sudo apt-get upgrade -y \
6464
python3-dev \
6565
python3-distutils \
6666
subversion \
67-
p7zip-full
67+
p7zip-full \
68+
&& sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
6869

6970
# install nodejs and npm (for plotly)
7071
# install pyenv https://github.com/pyenv/pyenv-installer
@@ -106,13 +107,8 @@ RUN curl -sL https://deb.nodesource.com/setup_12.x | sudo bash - \
106107
&& sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
107108

108109
# copying will cause rebuild at minimum to start from here
109-
COPY ./src "${PACKAGE_DIR}/src"
110-
COPY ./scripts "${PACKAGE_DIR}/scripts"
111-
COPY ./requirements_fixed.txt "${PACKAGE_DIR}/requirements_fixed.txt"
112-
COPY ./requirements_unfixed.txt "${PACKAGE_DIR}/requirements_unfixed.txt"
113-
COPY ./setup.py "${PACKAGE_DIR}/setup.py"
114-
COPY ./pytest.ini "${PACKAGE_DIR}/pytest.ini"
115-
COPY ./.test.env "${PACKAGE_DIR}/.test.env"
110+
# use .dockerignore to add files to docker image
111+
COPY ./ "${PACKAGE_DIR}"
116112

117113
# copied directory will not have user ownership by default
118114
# install energyplus versions desired in `scripts/setup/install_ep.sh`
@@ -125,17 +121,17 @@ RUN sudo chown -R "${USER_NAME}" "${PACKAGE_DIR}" \
125121
&& cd "${PACKAGE_DIR}" \
126122
&& ${PYENV_ROOT}/versions/3.8.6/bin/python3.8 -m venv "${LIB_DIR}/${VENV_NAME}" \
127123
&& . "${LIB_DIR}/${VENV_NAME}/bin/activate" \
128-
&& pip install --upgrade setuptools pip \
129-
# && pip install -r "requirements_fixed.txt" \
130-
&& pip install -r "requirements_unfixed.txt" \
124+
&& pip install --no-cache-dir --upgrade setuptools pip \
125+
# && pip install --no-cache-dir -r "requirements.txt" \
126+
&& pip install --no-cache-dir -r "requirements_unfixed.txt" \
131127
&& pip install --editable . \
132128
&& cd "${EXT_DIR}/PyFMI" \
133129
&& python "setup.py" install --fmil-home="${FMIL_HOME}" \
134130
&& cd "${EXT_DIR}" \
135131
&& wget "https://github.com/RJT1990/pyflux/archive/0.4.15.zip" \
136132
&& unzip "0.4.15.zip" && rm "0.4.15.zip" \
137133
&& cd "pyflux-0.4.15" \
138-
&& pip install .
134+
&& pip install --no-cache-dir .
139135

140136
# install jupyter lab extensions for plotly
141137
# if jupyter lab build fails with webpack optimization, set --minimize=False

‎README.md

+47-22
Original file line numberDiff line numberDiff line change
@@ -36,33 +36,58 @@ Copy the template files and fill in the variables mentioned below:
3636
```bash
3737
cp .env.template .env
3838
cp docker-compose.yml.template docker-compose.yml
39+
# and if you want to run the tests
40+
# .test.env does not need to be editted, unless you want to inject creds
41+
cp .test.env.template .test.env
3942
```
4043
Note: `docker-compose` behaviour may be slightly different on your host OS
41-
(Windows, Mac OS, Linux) with respect to how the expansion of environment
42-
variables works. If the base `docker-compose.yml` file fails on interpreting
43-
variables, try inlining those specific variables, e.g. replacing `${LOCAL_PACKAGE_DIR}`
44-
with `<where you cloned the repo to>/building-controls-simulator`.
44+
(Windows, Mac OS, Linux) and version of the CLI with respect to how the expansion of environment
45+
variables works. Make sure you have the correct version (mentioned above).
4546

4647
Edit in `.env.template`:
4748
```bash
4849
...
49-
LOCAL_PACKAGE_DIR=<where you cloned repo>
50-
...
51-
DYD_GCS_URI_BASE=<Donate your data Google Cloud Service bucket>
52-
DYD_METADATA_URI=<Donate your data meta_data file Google Cloud Service URI>
53-
NREL_DEV_API_KEY=<your key>
54-
NREL_DEV_EMAIL=<your email>
50+
LOCAL_PACKAGE_DIR=<where you cloned the repo>
5551
...
5652
```
5753

58-
Now you're ready to build and launch the container!
54+
Now you're ready to get the image and launch the container!
5955
If you delete the docker image just go through the setup here again to rebuild it.
6056

61-
##### Note: Docker images may use up to 12 GB of disk space - make sure you have this available before building.
62-
The size of the container image can be reduced to roughly 5 GB by not installing
57+
### Pull Docker image from Dockerhub
58+
59+
You can access the latest release image from: https://hub.docker.com/r/tstesco/building-controls-simulator/tags via CLI:
60+
61+
```bash
62+
docker pull tstesco/building-controls-simulator:0.3.3-alpha
63+
```
64+
65+
If you are using the Dockerhub repository make sure that your `.env` file contains
66+
the line
67+
```bash
68+
DOCKERHUB_REPOSITORY=tstesco
69+
```
70+
71+
This allows `docker-compose.yml` to find and use the correct image. Change this
72+
line in `docker-compose.yml` if you want to use a locally built image.
73+
74+
```yml
75+
# change this if want to build your own image
76+
image: ${DOCKERHUB_REPOSITORY}/${DOCKER_IMAGE}:${VERSION_TAG}
77+
```
78+
79+
to
80+
81+
```yml
82+
# change this if want to build your own image
83+
image: ${DOCKER_IMAGE}:${VERSION_TAG}
84+
```
85+
86+
##### Note: Locally built Docker images may use up to 10 GB of disk space - make sure you have this available before building.
87+
The size of the container image can be reduced to below 5 GB by not installing
6388
every EnergyPlus version in `scripts/setup/install_ep.sh` and not downloading
6489
all IECC 2018 IDF files in `scripts/setup/download_IECC_idfs.sh`. Simply comment
65-
out the files you do not need.
90+
out the versions/files you do not need in the respective files.
6691

6792
## Run BCS with Jupyter Lab Server (recommended: option 1)
6893

@@ -286,29 +311,29 @@ python -m pytest src/python
286311

287312
## Changing dependency versions
288313

289-
The dependencies are pinned to exact versions in the `requirements_fixed.txt` file.
314+
The dependencies are pinned to exact versions in the `requirements.txt` file.
290315
To change this simply change line (approx) 124 in the `Dockerfile` from:
291316
```
292-
&& pip install -r "requirements_fixed.txt" \
293-
# && pip install -r "requirements_unfixed.txt" \
317+
&& pip install --no-cache-dir -r "requirements.txt" \
318+
# && pip install --no-cache-dir -r "requirements_unfixed.txt" \
294319
```
295320

296321
to
297322

298323
```
299-
# && pip install -r "requirements_fixed.txt" \
300-
&& pip install -r "requirements_unfixed.txt" \
324+
# && pip install --no-cache-dir -r "requirements.txt" \
325+
&& pip install --no-cache-dir -r "requirements_unfixed.txt" \
301326
```
302327
303328
This will install the latest satisfying versions of all dependencies. After testing that
304-
the dependencies are working freeze them into a new `requirements_fixed.txt` file.
329+
the dependencies are working freeze them into a new `requirements.txt` file.
305330
306331
```
307-
pip freeze > requirements_fixed.txt
332+
pip freeze > requirements.txt
308333
```
309334
310335
Several dependencies are installed from source so these must be removed from the
311-
`requirements_fixed.txt` file. These are:
336+
`requirements.txt` file. These are:
312337
313338
```
314339
PyFMI

‎requirements_fixed.txt ‎requirements.txt

+9-11
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ click==7.1.2
2020
coverage==5.3.1
2121
cycler==0.10.0
2222
Cython==0.29.21
23-
dask==2020.12.0
23+
dask==2021.1.0
2424
decorator==4.4.2
2525
defusedxml==0.6.0
2626
docutils==0.16
@@ -30,14 +30,14 @@ flake8==3.8.4
3030
fsspec==0.8.5
3131
future==0.18.2
3232
gcsfs==0.7.1
33-
google-api-core==1.24.1
33+
google-api-core==1.25.0
3434
google-auth==1.24.0
3535
google-auth-oauthlib==0.4.2
3636
google-cloud-bigquery==2.3.1
3737
google-cloud-bigquery-storage==2.1.0
3838
google-cloud-core==1.5.0
3939
google-cloud-storage==1.35.0
40-
google-crc32c==1.1.0
40+
google-crc32c==1.1.1
4141
google-resumable-media==1.2.0
4242
googleapis-common-protos==1.52.0
4343
grpcio==1.34.1
@@ -72,7 +72,7 @@ munch==2.5.0
7272
mypy-extensions==0.4.3
7373
nbclient==0.5.1
7474
nbconvert==6.0.7
75-
nbformat==5.1.0
75+
nbformat==5.1.2
7676
nest-asyncio==1.4.3
7777
notebook==6.2.0
7878
numba==0.52.0
@@ -81,7 +81,7 @@ numexpr==2.7.2
8181
numpy==1.19.5
8282
oauthlib==3.1.0
8383
packaging==20.8
84-
pandas==1.2.0
84+
pandas==1.2.1
8585
pandas-gbq==0.14.1
8686
pandocfilters==1.4.3
8787
parso==0.8.1
@@ -93,7 +93,7 @@ Pillow==8.1.0
9393
plotly==4.14.3
9494
pluggy==0.13.1
9595
prometheus-client==0.9.0
96-
prompt-toolkit==3.0.10
96+
prompt-toolkit==3.0.11
9797
proto-plus==1.13.0
9898
protobuf==3.14.0
9999
psutil==5.8.0
@@ -107,23 +107,21 @@ pycparser==2.20
107107
pydata-google-auth==1.1.0
108108
pydot3k==1.0.17
109109
pyflakes==2.2.0
110-
pyflux==0.4.15
111-
PyFMI==2.7.4
112110
Pygments==2.7.4
113111
pyparsing==2.4.7
114112
pyrsistent==0.17.3
115113
pytest==6.2.1
116114
pytest-ordering==0.6
117115
python-dateutil==2.8.1
118116
pytz==2020.5
119-
PyYAML==5.3.1
120-
pyzmq==21.0.0
117+
PyYAML==5.4
118+
pyzmq==21.0.1
121119
regex==2020.11.13
122120
requests==2.25.1
123121
requests-oauthlib==1.3.0
124122
retrying==1.3.3
125123
rsa==4.7
126-
scikit-learn==0.24.0
124+
scikit-learn==0.24.1
127125
scipy==1.6.0
128126
Send2Trash==1.5.0
129127
Shapely==1.7.1

‎src/python/BuildingControlsSimulator/BuildingModels/EnergyPlusBuildingModel.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ def get_output_states(self):
9595
def get_model_name(self):
9696
# only need the idf file name because the weather is determined from
9797
# the combination of idf file and data_source-identifier
98-
return f"EnergyPlus_{self.idf.idf_name.rstrip('.idf')}"
98+
_model_name = f"EnergyPlus_{self.idf.idf_name.rstrip('.idf')}"
99+
_model_name = _model_name.replace(".", "_")
100+
return _model_name
99101

100102
@property
101103
def timesteps_per_hour(self):

‎src/python/BuildingControlsSimulator/ControllerModels/Deadband.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@ def get_output_states(self):
5656
]
5757

5858
def get_model_name(self):
59-
return f"Deadband_{self.deadband}".replace(".", "-")
59+
_model_name = f"Deadband_{self.deadband}"
60+
_model_name = _model_name.replace(".", "_")
61+
return _model_name
6062

6163
def initialize(
6264
self,

‎src/python/BuildingControlsSimulator/ControllerModels/FMIController.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,6 @@ class FMIController(ControllerModel):
2929
step_size_seconds = attr.ib()
3030

3131
def get_model_name(self):
32-
fmu_name = os.path.basename(self.fmu_path)
33-
return f"FMU_{fmu_name}"
32+
_model_name = os.path.basename(self.fmu_path)
33+
_model_name = _model_name.replace(".", "_")
34+
return _model_name

‎src/python/BuildingControlsSimulator/DataClients/DataClient.py

+10
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,16 @@ def get_data(self):
232232
[STATES.CALENDAR_EVENT],
233233
] = pd.NA
234234

235+
# finally convert dtypes to final types now that nulls in
236+
# non-nullable columns have been properly filled or removed
237+
_data = convert_spec(
238+
_data,
239+
src_spec=self.internal_spec,
240+
dest_spec=self.internal_spec,
241+
src_nullable=True,
242+
dest_nullable=False
243+
)
244+
235245
else:
236246
raise ValueError(
237247
f"ID={self.sim_config['identifier']} has no full_data_periods "

‎src/python/BuildingControlsSimulator/DataClients/DataDestination.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ def put_data(self, df, sim_name):
3434
pass
3535

3636
def get_file_name(self, sim_name):
37-
return f"{sim_name}.{self.file_extension}"
37+
# sim_name may contain . character, replace this safely
38+
safe_sim_name = sim_name.replace(".","_")
39+
return f"{safe_sim_name}.{self.file_extension}"
3840

3941
def get_local_cache_file(self, sim_name):
4042
if self.local_cache:
@@ -114,8 +116,6 @@ def write_data_by_extension(
114116
index=False,
115117
)
116118
elif file_extension == "csv.zip":
117-
raise NotImplementedError("Pandas 1.2.0 has issue with writting zip files.")
118-
# see
119119
_df.to_csv(
120120
filepath_or_buffer,
121121
compression="zip",

‎src/python/BuildingControlsSimulator/DataClients/DataSource.py

+6
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ def read_data_static(filepath_or_buffer, data_spec, extension="parquet.gzip"):
115115
if _col != data_spec.datetime_column
116116
],
117117
data_spec,
118+
src_nullable=True,
119+
dest_nullable=True,
118120
),
119121
)
120122
elif extension == "csv.zip":
@@ -129,6 +131,8 @@ def read_data_static(filepath_or_buffer, data_spec, extension="parquet.gzip"):
129131
if _col != data_spec.datetime_column
130132
],
131133
data_spec,
134+
src_nullable=True,
135+
dest_nullable=True,
132136
),
133137
)
134138
elif extension in ["csv.gzip", "csv.gz"]:
@@ -143,6 +147,8 @@ def read_data_static(filepath_or_buffer, data_spec, extension="parquet.gzip"):
143147
if _col != data_spec.datetime_column
144148
],
145149
data_spec,
150+
src_nullable=True,
151+
dest_nullable=True,
146152
),
147153
)
148154
else:

‎src/python/BuildingControlsSimulator/DataClients/DataSpec.py

+71-42
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,6 @@
1717
def spec_unit_conversion(df, src_spec, dest_spec):
1818
"""This method must be able to evaluate multiple sources should
1919
a channel be composed from multiple sources."""
20-
if src_spec == dest_spec:
21-
logger.info("spec_unit_conversion: src_spec is equal to dest_spec.")
22-
return df
23-
2420
for k, v in src_spec.full.spec.items():
2521
if k in df.columns:
2622
src_unit = v["unit"]
@@ -60,11 +56,38 @@ def spec_unit_conversion(df, src_spec, dest_spec):
6056
return df
6157

6258

63-
def get_dtype_mapper(df_cols, dest_spec):
59+
def get_dtype_mapper(df_cols, dest_spec, src_nullable=False, dest_nullable=False):
6460
# we only need to consider the destination spec
6561
dtype_mapper = {
6662
k: v["dtype"] for k, v in dest_spec.full.spec.items() if k in df_cols
6763
}
64+
65+
# convert between nullable columns and non-nullable for compatability
66+
if dest_nullable:
67+
for k,v in dtype_mapper.items():
68+
if v == "bool":
69+
dtype_mapper[k] = "boolean"
70+
elif v == "int8":
71+
dtype_mapper[k] = "Int8"
72+
elif v == "int16":
73+
dtype_mapper[k] = "Int16"
74+
elif v == "int32":
75+
dtype_mapper[k] = "Int32"
76+
elif v == "int64":
77+
dtype_mapper[k] = "Int64"
78+
else:
79+
for k,v in dtype_mapper.items():
80+
if v == "boolean":
81+
dtype_mapper[k] = "bool"
82+
elif v == "Int8":
83+
dtype_mapper[k] = "int8"
84+
elif v == "Int16":
85+
dtype_mapper[k] = "int16"
86+
elif v == "Int32":
87+
dtype_mapper[k] = "int32"
88+
elif v == "Int64":
89+
dtype_mapper[k] = "int64"
90+
6891
return dtype_mapper
6992

7093

@@ -120,7 +143,9 @@ def project_spec_keys(src_spec, dest_spec):
120143
return projection
121144

122145

123-
def convert_spec(df, src_spec, dest_spec, copy=False):
146+
def convert_spec(df, src_spec, dest_spec, src_nullable=False, dest_nullable=False, copy=False):
147+
# src_nullable: whether to use nullable int types
148+
# dest_nullable: whether to use nullable int types
124149
if type(src_spec) == type(dest_spec):
125150
logger.info("convert_spec: src_spec is equal to dest_spec.")
126151
return df
@@ -148,7 +173,7 @@ def convert_spec(df, src_spec, dest_spec, copy=False):
148173
_df = _df.rename(columns=get_rename_mapper(src_spec=src_spec, dest_spec=dest_spec))
149174

150175
_df = _df.astype(
151-
dtype=get_dtype_mapper(df_cols=_df.columns, dest_spec=dest_spec),
176+
dtype=get_dtype_mapper(df_cols=_df.columns, dest_spec=dest_spec, src_nullable=src_nullable, dest_nullable=dest_nullable),
152177
)
153178
_df = _df.sort_values(dest_spec.datetime_column, ascending=True)
154179
return _df
@@ -174,6 +199,10 @@ def dtypes_are_pandas(self, attribute, value):
174199
"Int16",
175200
"Int32",
176201
"Int64",
202+
"int8",
203+
"int16",
204+
"int32",
205+
"int64",
177206
"UInt8",
178207
"UInt16",
179208
"UInt32",
@@ -294,91 +323,91 @@ def __init__(self):
294323
spec={
295324
STATES.AUXHEAT1: {
296325
"name": "auxHeat1",
297-
"dtype": "Int16",
326+
"dtype": "int16",
298327
"channel": CHANNELS.EQUIPMENT,
299328
"unit": UNITS.SECONDS,
300329
},
301330
STATES.AUXHEAT2: {
302331
"name": "auxHeat2",
303-
"dtype": "Int16",
332+
"dtype": "int16",
304333
"channel": CHANNELS.EQUIPMENT,
305334
"unit": UNITS.SECONDS,
306335
},
307336
STATES.AUXHEAT3: {
308337
"name": "auxHeat3",
309-
"dtype": "Int16",
338+
"dtype": "int16",
310339
"channel": CHANNELS.EQUIPMENT,
311340
"unit": UNITS.SECONDS,
312341
},
313342
STATES.COMPCOOL1: {
314343
"name": "compCool1",
315-
"dtype": "Int16",
344+
"dtype": "int16",
316345
"channel": CHANNELS.EQUIPMENT,
317346
"unit": UNITS.SECONDS,
318347
},
319348
STATES.COMPCOOL2: {
320349
"name": "compCool2",
321-
"dtype": "Int16",
350+
"dtype": "int16",
322351
"channel": CHANNELS.EQUIPMENT,
323352
"unit": UNITS.SECONDS,
324353
},
325354
STATES.COMPHEAT1: {
326355
"name": "compHeat1",
327-
"dtype": "Int16",
356+
"dtype": "int16",
328357
"channel": CHANNELS.EQUIPMENT,
329358
"unit": UNITS.SECONDS,
330359
},
331360
STATES.COMPHEAT2: {
332361
"name": "compHeat2",
333-
"dtype": "Int16",
362+
"dtype": "int16",
334363
"channel": CHANNELS.EQUIPMENT,
335364
"unit": UNITS.SECONDS,
336365
},
337366
STATES.DEHUMIDIFIER: {
338367
"name": "dehumidifier",
339-
"dtype": "Int16",
368+
"dtype": "int16",
340369
"channel": CHANNELS.EQUIPMENT,
341370
"unit": UNITS.SECONDS,
342371
},
343372
STATES.ECONOMIZER: {
344373
"name": "economizer",
345-
"dtype": "Int16",
374+
"dtype": "int16",
346375
"channel": CHANNELS.EQUIPMENT,
347376
"unit": UNITS.SECONDS,
348377
},
349378
STATES.FAN: {
350379
"name": "fan",
351-
"dtype": "Int16",
380+
"dtype": "int16",
352381
"channel": CHANNELS.EQUIPMENT,
353382
"unit": UNITS.SECONDS,
354383
},
355384
STATES.FAN_STAGE_ONE: {
356385
"name": "fan1",
357-
"dtype": "Int16",
386+
"dtype": "int16",
358387
"channel": CHANNELS.EQUIPMENT,
359388
"unit": UNITS.SECONDS,
360389
},
361390
STATES.FAN_STAGE_TWO: {
362391
"name": "fan2",
363-
"dtype": "Int16",
392+
"dtype": "int16",
364393
"channel": CHANNELS.EQUIPMENT,
365394
"unit": UNITS.SECONDS,
366395
},
367396
STATES.FAN_STAGE_THREE: {
368397
"name": "fan3",
369-
"dtype": "Int16",
398+
"dtype": "int16",
370399
"channel": CHANNELS.EQUIPMENT,
371400
"unit": UNITS.SECONDS,
372401
},
373402
STATES.HUMIDIFIER: {
374403
"name": "humidifier",
375-
"dtype": "Int16",
404+
"dtype": "int16",
376405
"channel": CHANNELS.EQUIPMENT,
377406
"unit": UNITS.SECONDS,
378407
},
379408
STATES.VENTILATOR: {
380409
"name": "ventilator",
381-
"dtype": "Int16",
410+
"dtype": "int16",
382411
"channel": CHANNELS.EQUIPMENT,
383412
"unit": UNITS.SECONDS,
384413
},
@@ -616,73 +645,73 @@ def __init__(self):
616645
spec={
617646
"auxHeat1": {
618647
"internal_state": STATES.AUXHEAT1,
619-
"dtype": "Int16",
648+
"dtype": "int16",
620649
"channel": CHANNELS.EQUIPMENT,
621650
"unit": UNITS.SECONDS,
622651
},
623652
"auxHeat2": {
624653
"internal_state": STATES.AUXHEAT2,
625-
"dtype": "Int16",
654+
"dtype": "int16",
626655
"channel": CHANNELS.EQUIPMENT,
627656
"unit": UNITS.SECONDS,
628657
},
629658
"auxHeat3": {
630659
"internal_state": STATES.AUXHEAT3,
631-
"dtype": "Int16",
660+
"dtype": "int16",
632661
"channel": CHANNELS.EQUIPMENT,
633662
"unit": UNITS.SECONDS,
634663
},
635664
"compCool1": {
636665
"internal_state": STATES.COMPCOOL1,
637-
"dtype": "Int16",
666+
"dtype": "int16",
638667
"channel": CHANNELS.EQUIPMENT,
639668
"unit": UNITS.SECONDS,
640669
},
641670
"compCool2": {
642671
"internal_state": STATES.COMPCOOL2,
643-
"dtype": "Int16",
672+
"dtype": "int16",
644673
"channel": CHANNELS.EQUIPMENT,
645674
"unit": UNITS.SECONDS,
646675
},
647676
"compHeat1": {
648677
"internal_state": STATES.COMPHEAT1,
649-
"dtype": "Int16",
678+
"dtype": "int16",
650679
"channel": CHANNELS.EQUIPMENT,
651680
"unit": UNITS.SECONDS,
652681
},
653682
"compHeat2": {
654683
"internal_state": STATES.COMPHEAT2,
655-
"dtype": "Int16",
684+
"dtype": "int16",
656685
"channel": CHANNELS.EQUIPMENT,
657686
"unit": UNITS.SECONDS,
658687
},
659688
"dehumidifier": {
660689
"internal_state": STATES.DEHUMIDIFIER,
661-
"dtype": "Int16",
690+
"dtype": "int16",
662691
"channel": CHANNELS.EQUIPMENT,
663692
"unit": UNITS.SECONDS,
664693
},
665694
"economizer": {
666695
"internal_state": STATES.ECONOMIZER,
667-
"dtype": "Int16",
696+
"dtype": "int16",
668697
"channel": CHANNELS.EQUIPMENT,
669698
"unit": UNITS.SECONDS,
670699
},
671700
"fan": {
672701
"internal_state": STATES.FAN,
673-
"dtype": "Int16",
702+
"dtype": "int16",
674703
"channel": CHANNELS.EQUIPMENT,
675704
"unit": UNITS.SECONDS,
676705
},
677706
"humidifier": {
678707
"internal_state": STATES.HUMIDIFIER,
679-
"dtype": "Int16",
708+
"dtype": "int16",
680709
"channel": CHANNELS.EQUIPMENT,
681710
"unit": UNITS.SECONDS,
682711
},
683712
"ventilator": {
684713
"internal_state": STATES.VENTILATOR,
685-
"dtype": "Int16",
714+
"dtype": "int16",
686715
"channel": CHANNELS.EQUIPMENT,
687716
"unit": UNITS.SECONDS,
688717
},
@@ -860,49 +889,49 @@ def __init__(self):
860889
spec={
861890
"auxHeat1": {
862891
"internal_state": STATES.AUXHEAT1,
863-
"dtype": "Int16",
892+
"dtype": "int16",
864893
"channel": CHANNELS.EQUIPMENT,
865894
"unit": UNITS.SECONDS,
866895
},
867896
"auxHeat2": {
868897
"internal_state": STATES.AUXHEAT2,
869-
"dtype": "Int16",
898+
"dtype": "int16",
870899
"channel": CHANNELS.EQUIPMENT,
871900
"unit": UNITS.SECONDS,
872901
},
873902
"auxHeat3": {
874903
"internal_state": STATES.AUXHEAT3,
875-
"dtype": "Int16",
904+
"dtype": "int16",
876905
"channel": CHANNELS.EQUIPMENT,
877906
"unit": UNITS.SECONDS,
878907
},
879908
"compCool1": {
880909
"internal_state": STATES.COMPCOOL1,
881-
"dtype": "Int16",
910+
"dtype": "int16",
882911
"channel": CHANNELS.EQUIPMENT,
883912
"unit": UNITS.SECONDS,
884913
},
885914
"compCool2": {
886915
"internal_state": STATES.COMPCOOL2,
887-
"dtype": "Int16",
916+
"dtype": "int16",
888917
"channel": CHANNELS.EQUIPMENT,
889918
"unit": UNITS.SECONDS,
890919
},
891920
"compHeat1": {
892921
"internal_state": STATES.COMPHEAT1,
893-
"dtype": "Int16",
922+
"dtype": "int16",
894923
"channel": CHANNELS.EQUIPMENT,
895924
"unit": UNITS.SECONDS,
896925
},
897926
"compHeat2": {
898927
"internal_state": STATES.COMPHEAT2,
899-
"dtype": "Int16",
928+
"dtype": "int16",
900929
"channel": CHANNELS.EQUIPMENT,
901930
"unit": UNITS.SECONDS,
902931
},
903932
"fan": {
904933
"internal_state": STATES.FAN,
905-
"dtype": "Int16",
934+
"dtype": "int16",
906935
"channel": CHANNELS.EQUIPMENT,
907936
"unit": UNITS.SECONDS,
908937
},

‎src/python/BuildingControlsSimulator/DataClients/GBQDataSource.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def get_data(self, sim_config):
4545
if _data.empty:
4646
_data = self.get_gbq_data(sim_config, local_cache_file)
4747
_data = self.drop_unused_columns(_data=_data)
48-
_data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal())
48+
_data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal(), src_nullable=True, dest_nullable=True)
4949
return _data
5050

5151
def get_gbq_data(self, sim_config, local_cache_file):

‎src/python/BuildingControlsSimulator/DataClients/GCSDataSource.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def get_data(self, sim_config):
3939
if _data.empty:
4040
_data = self.get_gcs_cache(sim_config, local_cache_file)
4141
_data = self.drop_unused_columns(_data=_data)
42-
_data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal())
42+
_data = convert_spec(df=_data, src_spec=self.data_spec, dest_spec=Internal(), src_nullable=True, dest_nullable=True)
4343
return _data
4444

4545
@abstractmethod

‎src/python/BuildingControlsSimulator/DataClients/LocalSource.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,6 @@ def get_data(self, sim_config):
6161
_data = self.get_local_cache(local_cache_file)
6262
_data = self.drop_unused_columns(_data=_data)
6363
_data = convert_spec(
64-
df=_data, src_spec=self.data_spec, dest_spec=Internal(), copy=False
64+
df=_data, src_spec=self.data_spec, dest_spec=Internal(), copy=False, src_nullable=True, dest_nullable=True
6565
)
6666
return _data

‎src/python/BuildingControlsSimulator/DataClients/test_GCSDestination.py

+2
Original file line numberDiff line numberDiff line change
@@ -108,5 +108,7 @@ def test_put_data(self):
108108
r_df,
109109
src_spec=self.data_client.destination.data_spec,
110110
dest_spec=Internal(),
111+
src_nullable=True,
112+
dest_nullable=True,
111113
)
112114
assert _df.equals(cr_df)

‎src/python/BuildingControlsSimulator/DataClients/test_LocalDestination.py

+2
Original file line numberDiff line numberDiff line change
@@ -109,5 +109,7 @@ def test_put_data(self):
109109
r_df,
110110
src_spec=self.data_client.destination.data_spec,
111111
dest_spec=Internal(),
112+
src_nullable=True,
113+
dest_nullable=True,
112114
)
113115
assert _df.equals(cr_df)

‎src/python/BuildingControlsSimulator/Simulator/Simulation.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ def sim_name(self):
140140
_building_model_name = self.building_model.get_model_name()
141141
_controller_model_name = self.controller_model.get_model_name()
142142

143-
return "_".join(
143+
_sim_name = "_".join(
144144
[
145145
_prefix,
146146
_sim_run_identifier,
@@ -150,6 +150,9 @@ def sim_name(self):
150150
_controller_model_name,
151151
]
152152
)
153+
# safely remove any errant . characters breaking extension handling
154+
_sim_name.replace(".", "_")
155+
return _sim_name
153156

154157
def create_models(self, preprocess_check=False):
155158
# TODO: only have the building model that requires dynamic building

‎src/python/BuildingControlsSimulator/Simulator/params_test_Simulator.py

+86-11
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
test_params_gcs_dyd = []
2222
test_params_gbq_flatfiles = []
2323

24-
if os.environ.get("LOCAL_CACHE_DIR"):
24+
# if os.environ.get("LOCAL_CACHE_DIR"):
25+
if False:
2526
test_params_local = [
2627
{
2728
"config": {
@@ -119,7 +120,8 @@
119120
},
120121
]
121122

122-
if os.environ.get("DYD_GCS_URI_BASE"):
123+
# if os.environ.get("DYD_GCS_URI_BASE"):
124+
if False:
123125
test_params_gcs_dyd = [
124126
{
125127
"config": {
@@ -271,13 +273,86 @@
271273

272274
if os.environ.get("FLATFILES_GBQ_TABLE"):
273275
test_params_gbq_flatfiles = [
276+
# {
277+
# "config": {
278+
# "identifier": os.environ.get("TEST_GBQ_FF_IDENTIFIER"),
279+
# "latitude": 41.8781,
280+
# "longitude": -87.6298,
281+
# "start_utc": "2019-01-14",
282+
# "end_utc": "2019-01-18",
283+
# "min_sim_period": "1D",
284+
# "sim_step_size_seconds": 60,
285+
# "output_step_size_seconds": 300,
286+
# },
287+
# "data_client": {
288+
# "is_local_source": False,
289+
# "is_gcs_source": False,
290+
# "is_gbq_source": True,
291+
# "gcp_project": os.environ.get("DYD_GOOGLE_CLOUD_PROJECT"),
292+
# "gcs_uri_base": None,
293+
# "gbq_table": os.environ.get("FLATFILES_GBQ_TABLE"),
294+
# "source_data_spec": FlatFilesSpec(),
295+
# "source_local_cache": os.environ.get("LOCAL_CACHE_DIR"),
296+
# "is_local_destination": True,
297+
# "is_gcs_destination": False,
298+
# "is_gbq_destination": False,
299+
# "destination_data_spec": FlatFilesSpec(),
300+
# "destination_local_cache": os.environ.get("LOCAL_CACHE_DIR"),
301+
# },
302+
# "building_model": {
303+
# "is_energyplus_building": True,
304+
# "idf_name": "heatedbsmt_2story_2300sqft_gasfurnace_AC.idf",
305+
# "epw_name": "USA_IL_Chicago-OHare.Intl.AP.725300_TMY3.epw",
306+
# "building_config": {
307+
# "infiltration_ventilation": {
308+
# "ach50": 10,
309+
# "wsf": 0.6,
310+
# },
311+
# "insulation_r_si": {
312+
# "Exterior Roof": 1.0,
313+
# "Interior Ceiling": 6.7,
314+
# "Interior Floor": 0.75,
315+
# "Exterior Wall": 5.25,
316+
# "Exterior Floor": 5.0,
317+
# },
318+
# "windows": {
319+
# "u_factor": 0.8,
320+
# "solar_heat_gain": 0.30,
321+
# "visible_transmittance": 0.60,
322+
# },
323+
# "hvac": {
324+
# "heating_stages": 1,
325+
# "heating_equipment": "gas_furnace",
326+
# "heating_sizing_factor": 0.9,
327+
# "cooling_stages": 1,
328+
# "cooling_equipment": "dx_ac",
329+
# "cooling_sizing_factor": 0.9,
330+
# },
331+
# "thermal_mass": 1e7,
332+
# },
333+
# },
334+
# "controller_model": {
335+
# "is_deadband": True,
336+
# "is_fmu": False,
337+
# },
338+
# "state_estimator_model": {
339+
# "is_low_pass_filter": True,
340+
# "low_pass_filter_alpha": 0.2,
341+
# },
342+
# "expected_result": {
343+
# "mean_thermostat_temperature": 20.887479782104492,
344+
# "mean_thermostat_humidity": 20.74267578125,
345+
# "output_format_mean_thermostat_temperature": 695.9857788085938,
346+
# "output_format_mean_thermostat_humidity": 20.74267578125,
347+
# },
348+
# },
274349
{
275350
"config": {
276-
"identifier": os.environ.get("TEST_GBQ_FF_IDENTIFIER"),
277-
"latitude": 41.8781,
278-
"longitude": -87.6298,
279-
"start_utc": "2019-01-14",
280-
"end_utc": "2019-01-18",
351+
"identifier": os.environ.get("TEST_GBQ_FF_IDENTIFIER_2"),
352+
"latitude": 51.217373,
353+
"longitude": -114.296019 ,
354+
"start_utc": "2019-03-09",
355+
"end_utc": "2019-03-15",
281356
"min_sim_period": "1D",
282357
"sim_step_size_seconds": 60,
283358
"output_step_size_seconds": 300,
@@ -338,10 +413,10 @@
338413
"low_pass_filter_alpha": 0.2,
339414
},
340415
"expected_result": {
341-
"mean_thermostat_temperature": 20.887479782104492,
342-
"mean_thermostat_humidity": 20.74267578125,
343-
"output_format_mean_thermostat_temperature": 695.9857788085938,
344-
"output_format_mean_thermostat_humidity": 20.74267578125,
416+
"mean_thermostat_temperature": 20.319211959838867,
417+
"mean_thermostat_humidity": 16.502723693847656,
418+
"output_format_mean_thermostat_temperature": 685.7371826171875,
419+
"output_format_mean_thermostat_humidity": 16.502723693847656,
345420
},
346421
},
347422
]

‎src/python/BuildingControlsSimulator/StateEstimatorModels/LowPassFilter.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,9 @@ def get_output_states(self):
4646
]
4747

4848
def get_model_name(self):
49-
return f"Deadband_{self.deadband}".replace(".", "-")
49+
_model_name = "LowPass"
50+
_model_name = _model_name.replace(".", "_")
51+
return _model_name
5052

5153
def initialize(
5254
self,

0 commit comments

Comments
 (0)
Please sign in to comment.