Feature/podaac 4656 (#39)

* added collection list extraction to juputer_test WF

* fixed spacing

* added type to WF input

* removed token input

* expanded environment naming tolerance

* renamed collections to cumulus_api to avoid wrong imports

* updated name

* moved files to better positions

* adderd tokenfile location to workflow input

* some lint fix

* init lint disable

* lint fix

* lint...

* more lint

* updated F401 to F0401

* updated flake8 to ignore init.py

* updated workflow with new locations

* - Added cmr-association-diff import and usage
- Added secret usage to workflow
- Added FileHandler class

* removed \

* removed ./ from paths

* changed test execution only if input file exists

Co-authored-by: Zoltan Miskolci <zoltan.miskolci@jpl.nasa.gov>

.flake8

@@ -1,3 +1,4 @@
 [flake8]
 #ignore = ...
 max-line-length=180
+per-file-ignores = __init__.py:F401

.github/workflows/jupyter_test.yml

@@ -19,14 +19,20 @@ jobs:
           python-version: 3.9
       - name: Create prerequisites
         run: |
-          mkdir current_project
-          touch current_project/test_in.txt
-          echo "ASCATC-L2-Coastal" > current_project/test_in.txt
-          mkdir tests/jupyter/notebooks/output
+          mkdir jupyter_notebooks/output
       - name: Install dependencies
         run: |
-          pip install papermill
-      - name: Run Jupyter notebook
+          pip3 install --upgrade pip
+          pip3 install papermill
+          pip3 install xarray
+          pip3 install git+https://github.com/podaac/cmr-association-diff.git@6193079a14e36f4c9526aa426015c2b6be41f0e2
+      - name: Run CMR Association diff scripts
         run: |
-          python3 "./tests/jupyter/notebook_test.py" -n "./tests/jupyter/notebooks/harmony_concise_api_test.ipynb" -e uat -i ./current_project/test_in.txt -o ./tests/jupyter/notebooks//output
-     
+          cmr_association_diff -e uat -t service -a "cmr/uat_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o "jupyter_notebooks/output/uat_output_json.txt" --token ${{ secrets.LAUNCHPAD_TOKEN_UAT }}
+          cmr_association_diff -e ops -t service -a "cmr/ops_associations.txt" -p POCLOUD -n 'PODAAC Concise' -o "jupyter_notebooks/output/ops_output_json.txt" --token ${{ secrets.LAUNCHPAD_TOKEN_OPS }}
+      - name: Run Jupyter notebook on UAT
+        run: |
+          python3 "notebook_test.py" -n "jupyter_notebooks/harmony_concise_api_test.ipynb" -e uat -i "jupyter_notebooks/output/uat_output_json.txt" -o "jupyter_notebooks/output"
+      - name: Run Jupyter notebook on OPS
+        run: |
+          python3 "notebook_test.py" -n "jupyter_notebooks/harmony_concise_api_test.ipynb" -e ops -i "jupyter_notebooks/output/ops_output_json.txt" -o "jupyter_notebooks/output"

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+  - PODAAC-4653
+    - New github action workflow and Jupyter notebook that can be used for testing the concise service via Harmony
+  - PODAAC-4656
+    - Added cmr-association-diff import and usage
+    - Added secret usage to workflow
+    - Added FileHandler class
 ### Changed 
 - [issues/34](https://github.com/podaac/concise/issues/34): harmony-service-lib-py version updated to ^1.0.20 to support reading/writing STAC objects to/from S3
 ### Deprecated 

tests/jupyter/notebooks/harmony_concise_api_test.ipynb → jupyter_notebooks/harmony_concise_api_test.ipynb

@@ -9,6 +9,13 @@
     "# Harmony EOSS Concise API Tutorial"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -24,7 +31,6 @@
   {
    "cell_type": "markdown",
    "metadata": {
-    "jp-MarkdownHeadingCollapsed": true,
     "tags": []
    },
    "source": [
@@ -68,7 +74,6 @@
     "import xarray as xr\n",
     "import cmr\n",
     "import numpy as np\n",
-    "from podaac.subsetter import subset\n",
     "\n",
     "def setup_earthdata_login_auth(endpoint):\n",
     "    \"\"\"\n",
@@ -118,6 +123,13 @@
     "    return token"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -163,7 +175,7 @@
     "mode = cmr.queries.CMR_OPS\n",
     "\n",
     "# UAT Defaults\n",
-    "if venue == 'uat':\n",
+    "if venue.lower() == 'uat':\n",
     "    cmr_root = 'cmr.uat.earthdata.nasa.gov'\n",
     "    harmony_root = 'https://harmony.uat.earthdata.nasa.gov'\n",
     "    edl_root = 'uat.urs.earthdata.nasa.gov'\n",

notebook_test.py

@@ -0,0 +1,98 @@
+import papermill as pm
+import argparse
+
+from os import path
+
+from utils import FileHandler
+from utils.enums import Venue
+
+
+def parse_args():
+    """
+    Parses the program arguments
+    Returns
+    -------
+    args
+    """
+
+    parser = argparse.ArgumentParser(
+        description='Update CMR with latest profile',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument('-e', '--env',
+                        help='CMR environment used to pull results from.',
+                        required=True,
+                        choices=["uat", "ops", "ngap_uat", "ngap_ops"],
+                        metavar='uat or ops')
+
+    parser.add_argument('-n', '--notebook',
+                        help='Notebook to run',
+                        required=True,
+                        metavar='')
+
+    parser.add_argument('-i', '--input_file',
+                        help='File of json collections',
+                        required=True,
+                        metavar='')
+
+    parser.add_argument('-o', '--output_path',
+                        help='output path for success and fails',
+                        required=False,
+                        metavar='')
+
+    args = parser.parse_args()
+    return args
+
+def run():
+    """
+    Run from command line.
+
+    Returns
+    -------
+    """
+
+    _args = parse_args()
+    environment = _args.env
+    notebook = _args.notebook
+    inputFile = _args.input_file
+    output_location = _args.output_path if _args.output_path  else '.'
+
+    notebook_path = path.realpath(path.dirname(notebook))
+    notebook_name = path.basename(notebook)
+
+    success = []
+    fails = []
+
+    if path.exists(inputFile):
+        venue = Venue.from_str(environment)
+        collections = FileHandler.get_file_content_list_per_line(inputFile)
+        for collection in collections:
+
+            try:
+                print(collection)
+                pm.execute_notebook(
+                   notebook,
+                   f"{notebook_path}/output/{collection}_{environment}_output_{notebook_name}",
+                   parameters=dict(collection=collection, venue=venue.name)
+                )
+                success.append(collection)
+            except Exception as ex:
+                print(ex)
+                fails.append(collection)
+
+        # Create output files
+        if output_location:
+            success_outfile = path.realpath(f'{output_location}/{_args.env}_success.txt')
+            fail_outfile = path.realpath(f'{output_location}/{_args.env}_fail.txt')
+
+            if success:
+                with open(success_outfile, 'w') as the_file:
+                    the_file.writelines(x + '\n' for x in success)
+
+            if fails:
+                with open(fail_outfile, 'w') as the_file:
+                    the_file.writelines(x + '\n' for x in fails)
+
+if __name__ == '__main__':
+    run()

utils/__init__.py

@@ -0,0 +1,3 @@
+# pylint: disable=C0114
+
+from utils.filehandler import FileHandler

utils/enums/__init__.py

@@ -0,0 +1,3 @@
+# pylint: disable=C0114
+
+from utils.enums.venue import Venue

utils/enums/venue.py

@@ -0,0 +1,27 @@
+"""A Venue or Environment enum"""
+
+from enum import Enum
+
+
+class Venue(Enum):
+    """
+    Venue or Environment enum
+    """
+
+    UAT = 1
+    OPS = 2
+    SIT = 3
+
+    @staticmethod
+    def from_str(label):
+        """
+        Function to convert a string to Venue enum
+        """
+
+        if label.lower() in ["uat", "ngap_uat"]:
+            return Venue.UAT
+        if label.lower() in ["ops", "ngap_ops", "prod"]:
+            return Venue.OPS
+        if label.lower() in ["sit", "ngap_sit"]:
+            return Venue.SIT
+        raise NotImplementedError(f'No matching set up for env value "{label}"!')

utils/filehandler.py

@@ -0,0 +1,44 @@
+"""Module for file handling calls"""
+
+import json
+
+from os import path
+
+
+class FileHandler():
+    """
+    Class for file handling calls
+    """
+
+    def get_file_content(filename: str) -> str:
+        """
+        Function to get the content of a file
+        """
+
+        filepath = path.realpath(filename)
+        if path.exists(filepath):
+            result = open(filepath).read()
+            return result
+        else:
+            raise FileExistsError(f"{filepath} does not exists!")
+
+    def get_file_content_list_per_line(filename: str) -> list:
+        """
+        Function to get the content of a file, turned into a list.
+        Each line is a list element
+        """
+
+        filepath = path.realpath(filename)
+        if path.exists(filepath):
+            with open(filepath) as fileContent:
+                try:
+                    result = json.load(fileContent)
+                except ValueError:
+                    result = []
+                    fileContent.seek(0)
+                    lines = fileContent.readlines()
+                    for line in lines:
+                        result.append(line.strip())
+            return result
+        else:
+            raise FileExistsError(f"{filepath} does not exists!")