Merge pull request #29 from NASA-IMPACT/drop_pyhdf

Remove pyhdf dependency and use rasterio.

.dockerignore

@@ -2,6 +2,7 @@
 *.pyo
 __pycache__/
 venv/
+devenv/
 .vscode/
 .tox/
 dist/

.github/workflows/build_tox_tests.yml

@@ -6,11 +6,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
 
-      - name: Build image
-        run: docker build . --tag hls-metadata
-
-      - name: Tox tests in container
-        run: docker run -v $PWD:/hls-metadata hls-metadata
+      - name: Tox in docker
+        run: docker compose up --build

.gitignore

@@ -2,6 +2,7 @@
 *.pyo
 __pycache__/
 venv/
+devenv/
 .vscode/
 .tox/
 dist/

Dockerfile

@@ -1,18 +1,25 @@
 FROM osgeo/gdal:ubuntu-full-3.0.3
 
-# Required for click with Python 3.6
-ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
+RUN : \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+    build-essential \
+    libjpeg-dev \
+    zlib1g-dev \
+    python3-dev \
+    python3-pip \
+    python3-venv \
+    libhdf4-dev \
+    git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip3 install --no-cache --upgrade setuptools \
+    && pip3 install --no-cache rasterio==1.1.3 tox tox-venv --no-binary rasterio \
+    && :
 
-RUN apt-get update
-RUN apt-get install python3-pip python3-venv git -y
-
-
-RUN pip3 install rasterio==1.1.3 shapely --no-binary rasterio
-RUN apt-get install build-essential python3-dev python3-numpy libhdf4-dev -y
-RUN pip3 install tox tox-venv
-RUN pip3 install --upgrade setuptools
 RUN git clone https://github.com/NASA-IMPACT/hls-testing_data
 
-ENTRYPOINT ["/bin/sh", "-c"]
-CMD ["cd hls-metadata && tox -r"]
+WORKDIR /metadata_creator
+COPY ./ ./
+
+CMD ["tox", "-r", "-v"]

README.md

@@ -29,6 +29,7 @@ docker build -t hls-metadata . &&  docker run -v $PWD:/hls-metadata hls-metadata
 ### Development
 Because of the C lib dependencies on HDF4, developing this application is easisest in Docker.  To use the development container run
 ```bash
+docker compose up --build
 docker run -it -v $PWD:/hls-metadata hls-metadata /bin/bash
 ```
 At the container's shell prompt
@@ -44,3 +45,8 @@ cd hls-metadata
 pip3 install -e .
 create_metadata ../hls-testing_data/HLS.S30.T01LAH.2020097T222759.v1.5.hdf --save HLS.S30.T01LAH.2020097T222759.v1.5.xml
 ```
+
+To run the unit tests
+```bash
+docker compose up --build
+```

compose.yaml

@@ -0,0 +1,3 @@
+services:
+  tox:
+    build: .

metadata_creator/metadata_creator.py

@@ -8,7 +8,6 @@
 import click
 import rasterio
 from lxml import etree
-from pyhdf.SD import SD
 from pyproj import Transformer
 from rasterio import features
 from shapely import ops, wkt
@@ -101,13 +100,10 @@ def generate(self):
         return
 
     def extract_attributes(self):
-        # extract the attributes from the data file
-        try:
-            granule = SD(self.data_path, 1)
-        except TypeError:
-            granule = SD(self.data_path.encode("utf-8"), 1)
-        self.attributes = granule.attributes()
-        granule.end()
+        with rasterio.open(self.data_path) as dataset:
+            tags = dataset.tags()
+            print(tags)
+            self.attributes = tags
 
     def template_handler(self):
         """
@@ -146,50 +142,62 @@ def attribute_handler(self):
             "r",
         ) as attribute_file:
             attribute_mapping = json.load(attribute_file, object_pairs_hook=OrderedDict)
-        for attribute in self.root["AdditionalAttributes"]:
-            attribute_name = attribute_mapping[attribute["Name"]]
-            value = self.attributes.get(attribute_name, None)
-            if attribute_name == "NBAR_SOLAR_ZENITH" and value:
-                if not math.isnan(value):
-                    value = value
-                else:
-                    value = self.attributes.get("MEAN_SUN_ZENITH_ANGLE", None)
-            datatype = attribute["DataType"]
-            del attribute["DataType"]
-            del attribute["Description"]
-            if value is None and attribute.get("Values", None) is None:
-                if attribute["Name"] == "MGRS_TILE_ID":
-                    attribute["Values"] = {"Value": self.data_file.split(".")[2][1:]}
-                else:
-                    missing_values = {
-                        "INT": -9999,
-                        "FLOAT": -9999.9,
-                        "STRING": "Not Available",
-                    }
-                    attribute["Values"] = {"Value": missing_values[datatype]}
-                continue
-            elif attribute.get("Values", None) is not None:
-                continue
-
-            values = None
-            if isinstance(value, list):
-                values = value
-
-            if not values and datatype in ("FLOAT", "INT") and "," in str(value):
-                values = value.split(",")
-
-            if not values and ";" in str(value):
-                values = value.replace("; ", ";").split(";")
-
-            if not values:
-                values = [value]
-
-            if datatype == "FLOAT" and value is not None:
-                if values:
-                    values = [round(float(v), 8) for v in values]
-                else:
-                    value = round(float(value), 8)
-            attribute["Values"] = values
+            for attribute in self.root["AdditionalAttributes"]:
+                attribute_name = attribute_mapping[attribute["Name"]]
+                value = self.attributes.get(attribute_name, None)
+                if attribute_name == "NBAR_SOLAR_ZENITH" and value:
+                    try:
+                        value = float(value)
+                    except ValueError:
+                        print("Float conversion failed for NBAR_SOLAR_ZENITH")
+                    if not math.isnan(value):
+                        value = value
+                    else:
+                        value = self.attributes.get("MEAN_SUN_ZENITH_ANGLE", None)
+                datatype = attribute["DataType"]
+                del attribute["DataType"]
+                del attribute["Description"]
+                if value is None and attribute.get("Values", None) is None:
+                    if attribute["Name"] == "MGRS_TILE_ID":
+                        attribute["Values"] = {
+                            "Value": self.data_file.split(".")[2][1:]
+                        }
+                    else:
+                        missing_values = {
+                            "INT": -9999,
+                            "FLOAT": -9999.9,
+                            "STRING": "Not Available",
+                        }
+                        attribute["Values"] = {"Value": missing_values[datatype]}
+                    continue
+                elif attribute.get("Values", None) is not None:
+                    continue
+
+                values = None
+                if isinstance(value, list):
+                    values = value
+
+                if not values and datatype in ("FLOAT", "INT") and "," in str(value):
+                    values = value.split(",")
+
+                if not values and ";" in str(value):
+                    values = value.replace("; ", ";").split(";")
+
+                if not values:
+                    values = [value]
+
+                if datatype == "FLOAT" and value is not None:
+                    if values:
+                        values = [round(float(v), 8) for v in values]
+                    else:
+                        value = round(float(value), 8)
+
+                if datatype == "INT" and value is not None:
+                    if values:
+                        values = [int(v) for v in values]
+                    else:
+                        value = int(value)
+                attribute["Values"] = values
 
     def online_resource(self):
         """

setup.py

@@ -8,7 +8,6 @@
     install_requires=[
         "click~=7.1.0",
         "numpy",
-        "pyhdf",
         "pyproj==2.6.1",
         "rasterio",
         "shapely==1.8a1",