Skip to content

Commit

Permalink
Unit test for IONEX functionalities (#80)
Browse files Browse the repository at this point in the history
* Unit test for IONEX functionalities
* Add ionex unit test data
* adopt pytest fixtures
* Allocate variables in conftest
* rewrite Docker command
* Test with pytest in circleci
docker - moved copy of source after miniconda install
* fix tec URL
  • Loading branch information
vbrancat authored Feb 2, 2023
1 parent d3448d6 commit 8d04477
Show file tree
Hide file tree
Showing 8 changed files with 11,595 additions and 52 deletions.
6 changes: 5 additions & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ jobs:
- run:
name: "Build COMPASS docker image"
command: |
docker build . -f docker/Dockerfile
docker build . -f docker/Dockerfile -t test_image
- run:
name: "Run tests"
command: |
docker run test_image sh -c "cd ~/OPERA/COMPASS && pytest"
workflows:
build-workflow:
Expand Down
23 changes: 15 additions & 8 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,16 @@ RUN yum -y update &&\
yum -y install curl &&\
adduser compass_user

COPY . /home/compass_user/OPERA/COMPASS
RUN mkdir -p /home/compass_user/OPERA

RUN chown -R compass_user:compass_user /home/compass_user/OPERA &&\
chmod -R 755 /home/compass_user

USER compass_user
USER compass_user

ENV CONDA_PREFIX=/home/compass_user/miniconda3

# install Miniconda
WORKDIR /home/compass_user
RUN curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh &&\
bash miniconda.sh -b -p ${CONDA_PREFIX} &&\
Expand All @@ -25,19 +26,25 @@ RUN curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64
ENV PATH=${CONDA_PREFIX}/bin:${PATH}
RUN ${CONDA_PREFIX}/bin/conda init bash

# copy COMPASS and set compass_user as owner
COPY --chown=compass_user:compass_user . /home/compass_user/OPERA/COMPASS

# create CONDA environment
RUN conda create --name "COMPASS" --file /home/compass_user/OPERA/COMPASS/docker/specifile.txt

SHELL ["conda", "run", "-n", "COMPASS", "/bin/bash", "-c"]

RUN echo "Installing OPERA s1-reader" &&\
cd ${HOME}/OPERA &&\
curl -sSL https://github.com/opera-adt/s1-reader/archive/refs/tags/v0.1.5.tar.gz -o s1_reader_src.tar.gz &&\
WORKDIR /home/compass_user/OPERA

# installing OPERA s1-reader
RUN curl -sSL https://github.com/opera-adt/s1-reader/archive/refs/tags/v0.1.5.tar.gz -o s1_reader_src.tar.gz &&\
tar -xvf s1_reader_src.tar.gz &&\
ln -s s1-reader-0.1.5 s1-reader &&\
rm s1_reader_src.tar.gz &&\
python -m pip install ./s1-reader &&\
echo "Installing OPERA COMPASS" &&\
python -m pip install ./COMPASS &&\
python -m pip install ./s1-reader

# installing OPERA COMPASS
RUN python -m pip install ./COMPASS &&\
echo "conda activate COMPASS" >> /home/compass_user/.bashrc

WORKDIR /home/compass_user/scratch
Expand Down
69 changes: 38 additions & 31 deletions docker/specifile.txt
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h166bdaf_2.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.14-h166bdaf_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda
https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.17-h0b41bf4_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2
Expand Down Expand Up @@ -61,7 +62,7 @@ https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_openblas.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.47.0-hff17c54_1.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.51.0-hff17c54_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda
https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-ha49c73b_12.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.40.0-h753d276_0.tar.bz2
Expand All @@ -74,7 +75,7 @@ https://conda.anaconda.org/conda-forge/linux-64/readline-8.1.2-h0f457ee_0.tar.bz
https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h6239696_4.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h3eb15da_5.conda
https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.3-hafa529b_0.conda
https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda
Expand All @@ -83,63 +84,69 @@ https://conda.anaconda.org/conda-forge/linux-64/krb5-1.20.1-h81ceb04_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openblas.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libglib-2.74.1-h606061b_1.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h82bc61c_5.conda
https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h873f0b0_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h6adf6a1_2.conda
https://conda.anaconda.org/conda-forge/linux-64/nss-3.82-he02c5a1_0.conda
https://conda.anaconda.org/conda-forge/linux-64/python-3.11.0-ha86cf86_0_cpython.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/python-3.11.0-he550d4f_1_cpython.conda
https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.40.0-h4ff8645_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.7.2-h7f98852_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyh9f0ad1d_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/attrs-22.2.0-pyh71513ae_0.conda
https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.0-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.1-hc2a2eb6_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-h6ed2654_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.86.0-hdc1c0ab_2.conda
https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda
https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.87.0-hdc1c0ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_2.conda
https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.2-py311h14a6109_0.conda
https://conda.anaconda.org/conda-forge/linux-64/numpy-1.23.5-py311h7d28db0_0.conda
https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h7d73246_1.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/packaging-22.0-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_3.conda
https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.1-py311hbde0eaa_0.conda
https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda
https://conda.anaconda.org/conda-forge/noarch/packaging-23.0-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/noarch/pluggy-1.0.0-pyhd8ed1ab_5.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py311hd4cff14_5.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.7-py311h2582759_1.conda
https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py311h409f033_3.conda
https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.2.0-hd9d235c_0.conda
https://conda.anaconda.org/conda-forge/linux-64/curl-7.86.0-hdc1c0ab_2.conda
https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-nompi_h4df4325_100.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/curl-7.87.0-hdc1c0ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-nompi_h4df4325_101.conda
https://conda.anaconda.org/conda-forge/noarch/pip-22.3.1-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_2.conda
https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.0-h93bde94_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_3.conda
https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.1-h8ffa02c_2.conda
https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.1-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.21-py311hd4cff14_2.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/scipy-1.9.3-py311h69910c8_2.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.0-py311h0f577a2_0.conda
https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2
https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py311hd4cff14_1005.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/cryptography-38.0.4-py311h42a1071_0.conda
https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-ha76d385_4.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/cryptography-39.0.0-py311h9b4c7bb_0.conda
https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h7a142b4_6.conda
https://conda.anaconda.org/conda-forge/linux-64/h5py-3.7.0-nompi_py311hbe7f6d8_102.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.0-ha7026e8_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_h261ec11_106.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h7c8129e_22.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/poppler-22.12.0-h92391eb_0.conda
https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py311h7c961c7_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h221c8f1_23.conda
https://conda.anaconda.org/conda-forge/linux-64/poppler-23.01.0-h091648b_0.conda
https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py311h945b3ca_1.conda
https://conda.anaconda.org/conda-forge/linux-64/pyre-1.11.2-py311hbbb8f27_3.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.0-h3f4058f_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.1-hf2b5f72_1.conda
https://conda.anaconda.org/conda-forge/noarch/pyopenssl-22.1.0-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.1-py311hadb6153_1.conda
https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.13-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.2-hd532e3d_0.conda
https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-h8c90c07_5.conda
https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.0.0-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py311hadb6153_5.conda
https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda
https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2
https://conda.anaconda.org/conda-forge/linux-64/scipy-1.10.0-py311h8e6699e_0.conda
https://conda.anaconda.org/conda-forge/linux-64/isce3-0.9.0-py311h0802494_0.conda
https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2

2 changes: 2 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,7 @@ gdal>=3
lxml
pandas
pyproj
pytest
ruamel.yaml
scipy
yamale
18 changes: 6 additions & 12 deletions src/compass/utils/iono.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,15 @@
############################################################
# Links:
# IGS (NASA): https://cddis.nasa.gov/Data_and_Derived_Products/GNSS/atmospheric_products.html
# IMPC (DLR): https://impc.dlr.de/products/total-electron-content/near-real-time-tec/nrt-tec-global/
# IMPC (DLR): https://impc.dlr.de/products/total-electron-content/near-real-time-tec/near-real-time-tec-maps-global

import datetime as dt
import journal
import logging
import os
import re

import numpy as np
from scipy import interpolate
from compass.utils.helpers import get_module_name


def read_ionex(tec_file):
Expand Down Expand Up @@ -115,9 +114,6 @@ def get_ionex_value(tec_file, utc_sec, lat, lon,
tec_val: float or 1D np.ndarray
Vertical TEC value in TECU
'''
module_name = get_module_name(__file__)
error_channel = journal.error(f"{module_name}.get_ionex_value")

def interp_3d_rotate(interpfs, mins, lats, lons, utc_min, lat, lon):
ind0 = np.where((mins - utc_min) <= 0)[0][-1]
ind1 = ind0 + 1
Expand Down Expand Up @@ -168,7 +164,7 @@ def interp_3d_rotate(interpfs, mins, lats, lons, utc_min, lat, lon):
# option 1: interpolate between consecutive TEC maps
# testings shows better agreement with SAR obs than option 2.
tec_val = interpolate.interpn(
points=(mins, np.flip(lats), lons),
points=(mins, np.ascontiguousarray(np.flip(lats)), lons),
values=np.flip(tec_maps, axis=1),
xi=(utc_min, lat, lon),
method='linear',
Expand Down Expand Up @@ -208,7 +204,7 @@ def interp_3d_rotate(interpfs, mins, lats, lons, utc_min, lat, lon):
else:
msg = f'Un-recognized interp_method input: {interp_method}!'
msg += '\nSupported inputs: nearest, linear2d, linear3d.'
error_channel.log(msg)
logging.error(msg)
raise ValueError(msg)

return tec_val
Expand All @@ -234,8 +230,6 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'):
fname_dst_uncomp: str
Path to local uncompressed IONEX file
'''
module_name = get_module_name(__file__)
info_channel = journal.info(f"{module_name}.download_ionex")
# get the source (remote) and destination (local) file path/url
kwargs = dict(sol_code=sol_code, date_fmt=date_fmt)
fname_src = get_ionex_filename(date_str, tec_dir=None, **kwargs)
Expand All @@ -248,7 +242,7 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'):
cmd += ' --timestamping'

# Record executed command line in logging file
info_channel.log(f'Execute command: {cmd}')
logging.info(f'Execute command: {cmd}')

# download - run cmd in output dir
pwd = os.getcwd()
Expand All @@ -263,7 +257,7 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'):
or os.path.getmtime(fname_dst_uncomp) < os.path.getmtime(
fname_dst)):
cmd = f"gzip --force --keep --decompress {fname_dst}"
info_channel.log(f'Execute command: {cmd}')
logging.info(f'Execute command: {cmd}')
os.system(cmd)

return fname_dst_uncomp
Expand Down
50 changes: 50 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import os
import pytest
import types

from compass.utils import iono

@pytest.fixture(scope='session')
def ionex_params(download_data=True):
'''
Prepare IONEX data for unit test
Parameters
----------
download_data: bool
Boolean flag allow to download TEC data
for unit test if set to True
Returns
-------
tec_file: str
Path to local or downloaded TEC file to
use in the unit test
'''
test_params = types.SimpleNamespace()

# Set the path to fetch data for the test
test_params.tec_dir = os.path.join(os.path.dirname(__file__), "data")
test_params.date_str = '20151115'
test_params.sol_code = 'jpl'

# Create TEC directory
os.makedirs(test_params.tec_dir, exist_ok=True)

# Generate the TEC filename
test_params.tec_file = iono.get_ionex_filename(test_params.date_str,
tec_dir=test_params.tec_dir,
sol_code=test_params.sol_code)

# TODO figure out how to toggle download

# If prep_mode=True, download data
if download_data:
if not os.path.isfile(test_params.tec_file):
print(f'Download IONEX file at {test_params.date_str} from '
f'{test_params.sol_code} to {test_params.tec_dir}')
test_params.tec_file = iono.download_ionex(test_params.date_str,
test_params.tec_dir,
sol_code=test_params.sol_code)

return test_params
Loading

0 comments on commit 8d04477

Please sign in to comment.