Draft: Single test refactor

.github/workflows/docs.yml

@@ -13,13 +13,9 @@ on:
  - cron: 0 0 * * * # Scheduled run every day at midnight
 jobs:
  build:
-
- runs-on: ${{ matrix.platform }}
- strategy:
- fail-fast: false
- matrix:
- platform: [ubuntu-latest]
- python-version: [3.9]
+ runs-on: ubuntu-latest
+ container:
+ image: public.ecr.aws/w5r9l1c8/dev-swsoc-docker-lambda-base:latest
 
  steps:
  - uses: actions/checkout@v2

.github/workflows/testing.yml

@@ -35,12 +35,55 @@ jobs:
  pip install spacepy --no-build-isolation
  pip install -e .[test]
  if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }}
- - name: Run tests
+
+ - name: Install CDF library on Ubuntu
+ run: |
+ wget https://sdc-aws-support.s3.amazonaws.com/cdf-binaries/latest.zip
+ unzip latest.zip
+ echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV
+ if: ${{(matrix.platform == 'ubuntu-latest')}}
+
+ - name: Install CDF library on MacOS
+ run: |
+ wget https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/cdf39_0/macosx/CDF3_9_0-binary-signed.pkg
+ sudo installer -pkg CDF3_9_0-binary-signed.pkg -target /
+ ls -l
+ echo "CDF_LIB=cdf/lib" >> $GITHUB_ENV
+ if: ${{(matrix.platform == 'macos-latest')}}
+
+ # Set up CDF and run tests on Windows
+ - name: Install CDF on Windows
+ shell: cmd
+ run: |
+ # Download and unzip CDF
+ curl -L "https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/latest/windows/cdf3.9.0_64bit_WinZip_Installer.zip" --output cdf.zip
+ mkdir cdf_lib
+ tar -xf cdf.zip -C cdf_lib
+
+ # Set environment variables for CDF
+ set mydir=%cd%
+ set CDF_BASE=%mydir%\cdf_lib
+ set CDF_INC=%mydir%\cdf_lib\include
+ set CDF_LIB=%mydir%\cdf_lib\lib
+ set CDF_HELP=%mydir%\cdf_lib\help
+ set CDF_LEAPSECONDSTABLE=%mydir%\cdf_lib\CDFLeapSeconds.txt
+ set CLASSPATH=%mydir%\cdf_lib\CDFToolsDriver.jar;%mydir%\cdf_lib\lib\cdfjava.jar;%mydir%\cdf_lib\lib\cdfml.jar;%mydir%\cdf_lib\lib\cdfjson.jar;%mydir%\cdf_lib\lib\cdfj.jar;%mydir%\cdf_lib\lib\gson-2.8.6.jar;%mydir%\cdf_lib\lib\javax.json-1.0.4.jar;.
+ set PATH=%mydir%\cdf_lib;%mydir%\cdf_lib\bin;%PATH%
+ set TERMINFO=%mydir%\cdf_lib\lib\terminfo
+ set JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
+
+ # Run tests
+ pytest --pyargs hermes_eea --cov hermes_eea
+ # Skip Windows Python 3.11 tests until SpacePy is updated
+ if: ${{(matrix.platform == 'windows-latest' && matrix.python-version != '3.11')}}
+
+ - name: Run tests on Ubuntu and MacOS
  run: pytest --pyargs hermes_eea --cov hermes_eea
  env:
  PLATFORM: ${{ matrix.platform }}
  # Skip Windows Python 3.11 tests until SpacePy is updated
- if: ${{ !(matrix.platform == 'windows-latest' && matrix.python-version == '3.11') }}
+ if: ${{ !(matrix.platform == 'windows-latest') }}
+
  - name: Upload coverage reports to Codecov with GitHub Action
  uses: codecov/codecov-action@v3
  # Skip Windows Python 3.11 tests until SpacePy is updated

.readthedocs.yml

@@ -1,7 +1,6 @@
 # Read the Docs configuration file for Sphinx projects
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
-# Required
 version: 2
 
 # Set the OS, Python version and other tools you might need
@@ -21,4 +20,4 @@ build:
 # Optionally build your docs in additional formats such as PDF and ePub
 formats:
  - pdf
- - epub
+ - epub

docs/conf.py

@@ -8,6 +8,10 @@
 import os
 import sys
 
+# -- Environmental Variables ------------------------------------------------
+# Set CDF Library Path
+os.environ["CDF_LIB"] = "../cdf/lib"
+
 sys.path.insert(0, os.path.abspath(".."))
 # -- Project information -----------------------------------------------------
 

docs/whatsnew/changelog.rst

@@ -4,7 +4,5 @@
 Full Changelog
 **************
 
-.. changelog::
- :towncrier: ../../
- :towncrier-skip-if-empty:
- :changelog_file: ../../CHANGELOG.rst
+.. include:: ../../CHANGELOG.rst
+

hermes_eea/SkymapFactory.py

@@ -0,0 +1,122 @@
+import numpy as np
+from hermes_core import log
+from hermes_eea.io import EEA
+from hermes_eea.util.time import ccsds_to_cdf_time
+from hermes_eea import energies as voltages
+
+N_ENERGIES = 41
+N_DEFLECTIONS = 4
+N_AZIMUTH = 32
+
+
+# This may eventually be handled in a python multiprocessor module instance:
+def SkymapFactory(l0_cdf, energies, deflections, myEEA):
+ # ['Epoch', 'Epoch_plus_var', 'Epoch_minus_var', 'hermes_eea_step_counter',
+ # 'hermes_eea_counter1', 'hermes_eea_counter2', 'hermes_eea_accumulations',
+ # 'hermes_eea_sector_index', 'hermes_eea_sector_label'])
+
+ # science_data:
+ start_of_good_data = np.where(l0_cdf["SHEID"][:] == 1)[0][0]
+ integrates_at_end = np.where(
+ l0_cdf["SHEID"][start_of_good_data:] == 0
+ ) # has 63 values
+ # We are expecting integrates to be only at the beginning
+
+ # The Science Data:
+ stepper_table_packets = (np.where(l0_cdf["SHEID"][:] > 0))[0]
+ return_package = {}
+ beginning_packets = (
+ np.where((l0_cdf["STEP"][stepper_table_packets[0] :]) == 0)[0]
+ + stepper_table_packets[0]
+ )
+ package = []
+
+ epochs = ccsds_to_cdf_time.helpConvertEEA(l0_cdf)
+ try:
+ for ptr in range(0, len(beginning_packets)):
+ package.append(
+ (
+ l0_cdf["STEP"][beginning_packets[ptr] : beginning_packets[ptr + 1]],
+ l0_cdf["ACCUM"][
+ beginning_packets[ptr] : beginning_packets[ptr + 1]
+ ],
+ l0_cdf["COUNTER1"][
+ beginning_packets[ptr] : beginning_packets[ptr + 1]
+ ],
+ l0_cdf["COUNTER2"][
+ beginning_packets[ptr] : beginning_packets[ptr + 1]
+ ],
+ epochs[beginning_packets[ptr] : beginning_packets[ptr + 1]],
+ energies,
+ deflections,
+ ptr,
+ )
+ )
+ except IndexError:
+ log.info("Finished last interval")
+
+ result = []
+ for pckt in package:
+ packet_contents = do_eea_packet(*pckt)
+ if packet_contents is not None:
+ result.append(packet_contents)
+ myEEA.populate(myEEA, result)
+
+
+def do_eea_packet(
+ stepperTableCounter, counts, cnt1, cnt2, epoch, energies, deflections, ith_FSmap
+):
+ """
+ This function populates each sweep, or pass through
+ all of the energies and deflections designated by the stepper table
+
+ Parameters
+ ----------
+ stepperTableCounter - n_deflections * n_energies
+ counts - the structured arrays returned by CCSDSPY
+ cnt1 - the sum of this sweep's accum
+ cnt2 - same as above but +1...not clear yet
+ epoch - CDF Formatted time for every single measurement, [0] is the time for the sweep/packet
+ energies - so far we only have one stepper table with 41 energies
+ deflections - ...and 4 deflections
+ ith_FSmap - sweep counter
+
+ Returns
+ -------
+
+ """
+ return_package = {}
+ rows = len(stepperTableCounter)
+ # skymap is already full of zeros, why do it again?
+ # skymap = np.zeros((beginning_packets[ptr+1]-beginning_packets[ptr],N_AZIMUTH))
+ skymaps = []
+ pulse_a = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+ pulse_b = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+ counter1 = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+ counter2 = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+ usec = np.zeros((N_ENERGIES, N_DEFLECTIONS), dtype=np.uint16)
+
+ skymap = np.zeros((N_ENERGIES, N_DEFLECTIONS, N_AZIMUTH), dtype=np.uint16)
+
+ for row in stepperTableCounter:
+ dim0 = energies[row]
+ dim1 = deflections[row]
+ skymap[dim0, dim1, :] = counts[row, 0:N_AZIMUTH]
+ pulse_a[dim0, dim1] = counts[row][N_AZIMUTH]
+ pulse_b[dim0, dim1] = counts[row][N_AZIMUTH + 1]
+ counter1[dim0, dim1] = cnt1[row]
+ counter2[dim0, dim1] = cnt2[row]
+ usec[dim0, dim1] = epoch[row]
+
+ return_package["pulse_a"] = pulse_a
+ return_package["pulse_b"] = list(pulse_b)
+ return_package["counts"] = skymap
+ return_package["usec"] = usec
+ return_package["Epoch"] = epoch[0]
+ return_package["stats"] = np.sum(skymap)
+ return_package["energies"] = voltages
+ return_package["sun_angles"] = deflections
+ return_package["counter1"] = counter1
+ return_package["counter2"] = counter2
+
+ return return_package

hermes_eea/__init__.py

@@ -1,6 +1,8 @@
 # Licensed under Apache License v2 - see LICENSE.rst
 import os.path
+import sys
 
+sys.path.append(os.getcwd())
 from hermes_core import log
 from hermes_eea.io.file_tools import read_file
 
@@ -21,5 +23,54 @@
 
 _package_directory = os.path.dirname(os.path.abspath(__file__))
 _data_directory = os.path.abspath(os.path.join(_package_directory, "data"))
+_calibration_directory = os.path.abspath(os.path.join(_data_directory, "calibration"))
+
 
 log.info(f"hermes_eea version: {__version__}")
+
+stepper_table = "flight_stepper.txt"
+
+
+energies = [
+ 2.18000000e00,
+ 2.63177330e00,
+ 3.17717004e00,
+ 3.83559233e00,
+ 4.63046306e00,
+ 5.59005918e00,
+ 6.74851766e00,
+ 8.14704980e00,
+ 9.83540739e00,
+ 1.18736525e01,
+ 1.43342944e01,
+ 1.73048684e01,
+ 2.08910507e01,
+ 2.52204172e01,
+ 3.04469818e01,
+ 3.67566761e01,
+ 4.43739626e01,
+ 5.35698211e01,
+ 6.46713874e01,
+ 7.80735920e01,
+ 9.42532085e01,
+ 1.13785815e02,
+ 1.37366271e02,
+ 1.65833433e02,
+ 2.00200000e02,
+ 2.39800000e02,
+ 3.17794829e02,
+ 4.21157437e02,
+ 5.58138682e02,
+ 7.39673007e02,
+ 9.80251281e02,
+ 1.29907752e03,
+ 1.72160182e03,
+ 2.28155195e03,
+ 3.02362557e03,
+ 4.00705827e03,
+ 5.31035195e03,
+ 7.03754125e03,
+ 9.32649800e03,
+ 1.23599368e04,
+ 1.63800000e04,
+]