Unit test for IONEX functionalities

.circleci/config.yml

@@ -10,7 +10,12 @@ jobs:
       - run:
           name: "Build COMPASS docker image"
           command: |
-            docker build . -f docker/Dockerfile
+            docker build . -f docker/Dockerfile -t test_image
+      - run:
+          name: "Run tests"
+          command: |
+            docker run test_image sh -c "cd ~/OPERA/COMPASS &&\
+                                    python -m pytest"
 
 workflows:
   build-workflow:

docker/Dockerfile

@@ -13,7 +13,7 @@ COPY . /home/compass_user/OPERA/COMPASS
 RUN chown -R compass_user:compass_user /home/compass_user/OPERA &&\
     chmod -R 755 /home/compass_user
 
-USER compass_user 
+USER compass_user
 
 ENV CONDA_PREFIX=/home/compass_user/miniconda3
 

docker/specifile.txt

@@ -33,7 +33,8 @@ https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h166bdaf_2.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.14-h166bdaf_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.17-h0b41bf4_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2
@@ -61,7 +62,7 @@ https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_openblas.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.47.0-hff17c54_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.51.0-hff17c54_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-ha49c73b_12.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.40.0-h753d276_0.tar.bz2
@@ -74,7 +75,7 @@ https://conda.anaconda.org/conda-forge/linux-64/readline-8.1.2-h0f457ee_0.tar.bz
 https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.3-hd9c2040_1000.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-h166bdaf_4.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h6239696_4.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-h3eb15da_5.conda
 https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.3-hafa529b_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda
@@ -83,63 +84,69 @@ https://conda.anaconda.org/conda-forge/linux-64/krb5-1.20.1-h81ceb04_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-16_linux64_openblas.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/libglib-2.74.1-h606061b_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-16_linux64_openblas.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.4.0-h82bc61c_5.conda
-https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h873f0b0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h6adf6a1_2.conda
 https://conda.anaconda.org/conda-forge/linux-64/nss-3.82-he02c5a1_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/python-3.11.0-ha86cf86_0_cpython.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/python-3.11.0-he550d4f_1_cpython.conda
 https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.40.0-h4ff8645_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.7.2-h7f98852_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyh9f0ad1d_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/attrs-22.2.0-pyh71513ae_0.conda
 https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/certifi-2022.12.7-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-2.1.1-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.1-hc2a2eb6_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-h6ed2654_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.86.0-hdc1c0ab_2.conda
+https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda
+https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.87.0-hdc1c0ab_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_2.conda
-https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.2-py311h14a6109_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/numpy-1.23.5-py311h7d28db0_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h7d73246_1.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/packaging-22.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libpq-15.1-hb675445_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.1-py311hbde0eaa_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda
+https://conda.anaconda.org/conda-forge/noarch/packaging-23.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pluggy-1.0.0-pyhd8ed1ab_5.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py311hd4cff14_5.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml.clib-0.2.7-py311h2582759_1.conda
-https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/wheel-0.38.4-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h7f98852_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.10-h7f98852_1003.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py311h409f033_3.conda
 https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.2.0-hd9d235c_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/curl-7.86.0-hdc1c0ab_2.conda
-https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-nompi_h4df4325_100.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/curl-7.87.0-hdc1c0ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-nompi_h4df4325_101.conda
 https://conda.anaconda.org/conda-forge/noarch/pip-22.3.1-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_2.conda
-https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.0-h93bde94_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.1-h3248436_3.conda
+https://conda.anaconda.org/conda-forge/linux-64/proj-9.1.1-h8ffa02c_2.conda
+https://conda.anaconda.org/conda-forge/noarch/pytest-7.2.1-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/ruamel.yaml-0.17.21-py311hd4cff14_2.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/scipy-1.9.3-py311h69910c8_2.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.0-py311h0f577a2_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h55805fa_1.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/brotlipy-0.7.0-py311hd4cff14_1005.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/cryptography-38.0.4-py311h42a1071_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-ha76d385_4.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/cryptography-39.0.0-py311h9b4c7bb_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h7a142b4_6.conda
 https://conda.anaconda.org/conda-forge/linux-64/h5py-3.7.0-nompi_py311hbe7f6d8_102.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.0-ha7026e8_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_h261ec11_106.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h7c8129e_22.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/poppler-22.12.0-h92391eb_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py311h7c961c7_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h221c8f1_23.conda
+https://conda.anaconda.org/conda-forge/linux-64/poppler-23.01.0-h091648b_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py311h945b3ca_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/pyre-1.11.2-py311hbbb8f27_3.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.0-h3f4058f_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.1-hf2b5f72_1.conda
-https://conda.anaconda.org/conda-forge/noarch/pyopenssl-22.1.0-pyhd8ed1ab_0.tar.bz2
-https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.1-py311hadb6153_1.conda
-https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.13-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.2-hd532e3d_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-h8c90c07_5.conda
+https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.0.0-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py311hadb6153_5.conda
+https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/scipy-1.10.0-py311h8e6699e_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/isce3-0.9.0-py311h0802494_0.conda
-https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2
-

requirements.txt

@@ -4,5 +4,7 @@ gdal>=3
 #isce3        # since the conda-installed isce3 is not the most updated version, installing isce3 from stratch is recommended, to stay in sync with isce3 development.
 #journal      # as of Mar 2022, journal from conda does not support python3.9; since it is included during isce3 installation above, comment this out temporarily.
 pyproj
+pytest
 ruamel.yaml
+scipy
 yamale

src/compass/utils/iono.py

@@ -9,7 +9,7 @@
 #   IMPC (DLR): https://impc.dlr.de/products/total-electron-content/near-real-time-tec/nrt-tec-global/
 
 import datetime as dt
-import journal
+import logging
 import os
 import re
 
@@ -115,9 +115,6 @@ def get_ionex_value(tec_file, utc_sec, lat, lon,
     tec_val: float or 1D np.ndarray
         Vertical TEC value in TECU
     '''
-    module_name = get_module_name(__file__)
-    error_channel = journal.error(f"{module_name}.get_ionex_value")
-
     def interp_3d_rotate(interpfs, mins, lats, lons, utc_min, lat, lon):
         ind0 = np.where((mins - utc_min) <= 0)[0][-1]
         ind1 = ind0 + 1
@@ -168,7 +165,7 @@ def interp_3d_rotate(interpfs, mins, lats, lons, utc_min, lat, lon):
             # option 1: interpolate between consecutive TEC maps
             # testings shows better agreement with SAR obs than option 2.
             tec_val = interpolate.interpn(
-                points=(mins, np.flip(lats), lons),
+                points=(mins, np.ascontiguousarray(np.flip(lats)), lons),
                 values=np.flip(tec_maps, axis=1),
                 xi=(utc_min, lat, lon),
                 method='linear',
@@ -234,8 +231,6 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'):
     fname_dst_uncomp: str
         Path to local uncompressed IONEX file
     '''
-    module_name = get_module_name(__file__)
-    info_channel = journal.info(f"{module_name}.download_ionex")
     # get the source (remote) and destination (local) file path/url
     kwargs = dict(sol_code=sol_code, date_fmt=date_fmt)
     fname_src = get_ionex_filename(date_str, tec_dir=None, **kwargs)
@@ -248,7 +243,7 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'):
         cmd += ' --timestamping'
 
     # Record executed command line in logging file
-    info_channel.log(f'Execute command: {cmd}')
+    logging.info(f'Execute command: {cmd}')
 
     # download - run cmd in output dir
     pwd = os.getcwd()
@@ -263,7 +258,7 @@ def download_ionex(date_str, tec_dir, sol_code='jpl', date_fmt='%Y%m%d'):
             or os.path.getmtime(fname_dst_uncomp) < os.path.getmtime(
                 fname_dst)):
         cmd = f"gzip --force --keep --decompress {fname_dst}"
-        info_channel.log(f'Execute command: {cmd}')
+        logging.info(f'Execute command: {cmd}')
         os.system(cmd)
 
     return fname_dst_uncomp

tests/conftest.py

@@ -0,0 +1,50 @@
+import os
+import pytest
+import types
+
+from compass.utils import iono
+
+@pytest.fixture(scope='session')
+def ionex_params(download_data=True):
+    '''
+    Prepare IONEX data for unit test
+
+    Parameters
+    ----------
+    download_data: bool
+        Boolean flag allow to download TEC data
+        for unit test if set to True
+
+    Returns
+    -------
+    tec_file: str
+        Path to local or downloaded TEC file to
+        use in the unit test
+    '''
+    test_params = types.SimpleNamespace()
+
+    # Set the path to fetch data for the test
+    test_params.tec_dir = os.path.join(os.path.dirname(__file__), "data")
+    test_params.date_str = '20151115'
+    test_params.sol_code = 'jpl'
+
+    # Create TEC directory
+    os.makedirs(test_params.tec_dir, exist_ok=True)
+
+    # Generate the TEC filename
+    test_params.tec_file = iono.get_ionex_filename(test_params.date_str,
+                                          tec_dir=test_params.tec_dir,
+                                          sol_code=test_params.sol_code)
+
+    # TODO figure out how to toggle download
+
+    # If prep_mode=True, download data
+    if download_data:
+        if not os.path.isfile(test_params.tec_file):
+            print(f'Download IONEX file at {test_params.date_str} from '
+                  f'{test_params.sol_code} to {test_params.tec_dir}')
+            test_params.tec_file = iono.download_ionex(test_params.date_str,
+                                                       test_params.tec_dir,
+                                                       sol_code=test_params.sol_code)
+
+    return test_params