Release V 0.14.2

.github/workflows/changelog.yml

@@ -13,6 +13,6 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/create-jira-issue.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   call-create-jira-issue-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0
     secrets:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}

.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0

.github/workflows/release-template-comment.yml

@@ -7,6 +7,6 @@ on:
 
 jobs:
   call-release-checklist-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0
     with:
       release_prefix: HyP3 autoRIFT
     secrets:

.github/workflows/static-analysis.yml

@@ -4,10 +4,10 @@ on: push
 
 jobs:
   call-flake8-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.0
     with:
       local_package_names: hyp3_autorift
       excludes: src/hyp3_autorift/vend
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0

.github/workflows/tag-version.yml

@@ -7,6 +7,6 @@ on:
 
 jobs:
   call-bump-version-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/test-and-build.yml

@@ -12,18 +12,18 @@ on:
 
 jobs:
   call-pytest-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0
     with:
       local_package_name: hyp3_autorift
       python_versions: >-
         ["3.9"]
 
   call-version-info-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.0
 
   call-docker-ghcr-workflow:
     needs: call-version-info-workflow
-    uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.10.0
+    uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.0
     with:
       version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }}
     secrets:

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.15.0]
+### Added
+* `--publish-bucket` option has been added to the HyP3 entry point to additionally publish products an AWS bucket, such as the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`.
+* `upload_file_to_s3_with_publish_access_keys` to perform S3 uploads using credentials from the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment vairables.
 
 ## [0.14.1]
 ### Changed

src/hyp3_autorift/process.py

@@ -27,7 +27,7 @@
 
 from hyp3_autorift import geometry, image, io
 from hyp3_autorift.crop import crop_netcdf_product
-from hyp3_autorift.utils import get_esa_credentials
+from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_publish_access_keys
 
 log = logging.getLogger(__name__)
 
@@ -48,6 +48,16 @@
 DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \
                          'autorift_parameters/v001/autorift_landice_0120m.shp'
 
+PLATFORM_SHORTNAME_LONGNAME_MAPPING = {
+    'S1': 'sentinel1',
+    'S2': 'sentinel2',
+    'L4': 'landsatOLI',
+    'L5': 'landsatOLI',
+    'L7': 'landsatOLI',
+    'L8': 'landsatOLI',
+    'L9': 'landsatOLI',
+}
+
 
 def get_lc2_stac_json_key(scene_name: str) -> str:
     platform = get_platform(scene_name)
@@ -319,6 +329,48 @@ def apply_landsat_filtering(reference_path: str, secondary_path: str) \
     return reference_path, reference_zero_path, secondary_path, secondary_zero_path
 
 
+def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]:
+    with Dataset(ncfile) as ds:
+        var = ds.variables['img_pair_info']
+        return var.latitude, var.longitude
+
+
+def point_to_region(lat: float, lon: float) -> str:
+    """
+    Returns a string (for example, N78W124) of a region name based on
+    granule center point lat,lon
+    """
+    nw_hemisphere = 'N' if lat >= 0.0 else 'S'
+    ew_hemisphere = 'E' if lon >= 0.0 else 'W'
+
+    region_lat = int(10*np.trunc(np.abs(lat/10.0)))
+    if region_lat == 90:  # if you are exactly at a pole, put in lat = 80 bin
+        region_lat = 80
+
+    region_lon = int(10*np.trunc(np.abs(lon/10.0)))
+
+    if region_lon >= 180:  # if you are at the dateline, back off to the 170 bin
+        region_lon = 170
+
+    return f'{nw_hemisphere}{region_lat:02d}{ew_hemisphere}{region_lon:03d}'
+
+
+def get_opendata_prefix(file: Path):
+    # filenames have form GRANULE1_X_GRANULE2
+    scene = file.name.split('_X_')[0]
+
+    platform_shortname = get_platform(scene)
+    lat, lon = get_lat_lon_from_ncfile(file)
+    region = point_to_region(lat, lon)
+
+    return '/'.join([
+        'velocity_image_pair',
+        PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname],
+        'v02',
+        region
+    ])
+
+
 def process(
     reference: str,
     secondary: str,
@@ -520,6 +572,9 @@ def main():
     )
     parser.add_argument('--bucket', help='AWS bucket to upload product files to')
     parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files')
+    parser.add_argument('--publish-bucket', default='',
+                        help='Additionally, publish products to this bucket. Necessary credentials must be provided '
+                             'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.')
     parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE,
@@ -538,9 +593,15 @@ def main():
     g1, g2 = sorted(args.granules, key=get_datetime)
 
     product_file, browse_file = process(g1, g2, parameter_file=args.parameter_file, naming_scheme=args.naming_scheme)
+    thumbnail_file = create_thumbnail(browse_file)
 
     if args.bucket:
         upload_file_to_s3(product_file, args.bucket, args.bucket_prefix)
         upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix)
-        thumbnail_file = create_thumbnail(browse_file)
         upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix)
+
+    if args.publish_bucket:
+        prefix = get_opendata_prefix(product_file)
+        upload_file_to_s3_with_publish_access_keys(product_file, args.publish_bucket, prefix)
+        upload_file_to_s3_with_publish_access_keys(browse_file, args.publish_bucket, prefix)
+        upload_file_to_s3_with_publish_access_keys(thumbnail_file, args.publish_bucket, prefix)

src/hyp3_autorift/utils.py

@@ -1,9 +1,13 @@
+import logging
 import netrc
 import os
 from pathlib import Path
 from platform import system
 from typing import Tuple
 
+import boto3
+from hyp3lib.aws import get_content_type, get_tag_set
+
 
 ESA_HOST = 'dataspace.copernicus.eu'
 
@@ -28,3 +32,25 @@ def get_esa_credentials() -> Tuple[str, str]:
         "Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the "
         "ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file."
     )
+
+
+def upload_file_to_s3_with_publish_access_keys(path_to_file: Path, bucket: str, prefix: str = ''):
+    try:
+        access_key_id = os.environ['PUBLISH_ACCESS_KEY_ID']
+        access_key_secret = os.environ['PUBLISH_SECRET_ACCESS_KEY']
+    except KeyError:
+        raise ValueError(
+            'Please provide S3 Bucket upload access key credentials via the '
+            'PUBLISH_ACCESS_KEY_ID and PUBLISH_SECRET_ACCESS_KEY environment variables'
+        )
+
+    s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret)
+    key = str(Path(prefix) / path_to_file.name)
+    extra_args = {'ContentType': get_content_type(key)}
+
+    logging.info(f'Uploading s3://{bucket}/{key}')
+    s3_client.upload_file(str(path_to_file), bucket, key, extra_args)
+
+    tag_set = get_tag_set(path_to_file.name)
+
+    s3_client.put_object_tagging(Bucket=bucket, Key=key, Tagging=tag_set)

tests/test_process.py

@@ -1,5 +1,6 @@
 import io
 from datetime import datetime
+from pathlib import Path
 from re import match
 from unittest import mock
 from unittest.mock import MagicMock, patch
@@ -415,3 +416,27 @@ def mock_apply_filter_function(scene, _):
         process.apply_landsat_filtering('LT04', 'LE07')
     assert process.apply_landsat_filtering('LT04', 'LT05') == ('LT04', None, 'LT05', None)
     assert process.apply_landsat_filtering('LT04', 'LT04') == ('LT04', None, 'LT04', None)
+
+
+def test_point_to_prefix():
+    assert process.point_to_region(63.0, 128.0) == 'N60E120'
+    assert process.point_to_region(-63.0, 128.0) == 'S60E120'
+    assert process.point_to_region(63.0, -128.0) == 'N60W120'
+    assert process.point_to_region(-63.0, -128.0) == 'S60W120'
+    assert process.point_to_region(0.0, 128.0) == 'N00E120'
+    assert process.point_to_region(0.0, -128.0) == 'N00W120'
+    assert process.point_to_region(63.0, 0.0) == 'N60E000'
+    assert process.point_to_region(-63.0, 0.0) == 'S60E000'
+    assert process.point_to_region(0.0, 0.0) == 'N00E000'
+
+
+def test_get_lat_lon_from_ncfile():
+    file = Path('tests/data/'
+                'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc')
+    assert process.get_lat_lon_from_ncfile(file) == (-81.49, -128.28)
+
+
+def test_get_opendata_prefix():
+    file = Path('tests/data/'
+                'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc')
+    assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120'

tests/test_utils.py

@@ -1,6 +1,6 @@
 import pytest
 
-from hyp3_autorift.utils import ESA_HOST, get_esa_credentials
+from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_publish_access_keys
 
 
 def test_get_esa_credentials_env(tmp_path, monkeypatch):
@@ -45,3 +45,19 @@ def test_get_esa_credentials_missing(tmp_path, monkeypatch):
         msg = 'Please provide.*'
         with pytest.raises(ValueError, match=msg):
             get_esa_credentials()
+
+
+def test_upload_file_to_s3_credentials_missing(tmp_path, monkeypatch):
+    with monkeypatch.context() as m:
+        m.delenv('PUBLISH_ACCESS_KEY_ID', raising=False)
+        m.setenv('PUBLISH_SECRET_ACCESS_KEY', 'publish_access_key_secret')
+        msg = 'Please provide.*'
+        with pytest.raises(ValueError, match=msg):
+            upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')
+
+    with monkeypatch.context() as m:
+        m.setenv('PUBLISH_ACCESS_KEY_ID', 'publish_access_key_id')
+        m.delenv('PUBLISH_SECRET_ACCESS_KEY', raising=False)
+        msg = 'Please provide.*'
+        with pytest.raises(ValueError, match=msg):
+            upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')