From de15efcfad2c44be12100717429e3f41130f4179 Mon Sep 17 00:00:00 2001 From: Fabrice Brito Date: Thu, 19 Oct 2023 13:15:37 +0000 Subject: [PATCH] adds cloud native vs staged --- .github/workflows/build.yaml | 17 +- water-bodies/app-package-cloud-native.cwl | 264 ++++++++++++++++++++ water-bodies/app-package.cwl | 8 +- water-bodies/command-line-tools/crop/app.py | 12 +- 4 files changed, 293 insertions(+), 8 deletions(-) create mode 100644 water-bodies/app-package-cloud-native.cwl diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 82b2daf..f20bbb4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -4,6 +4,7 @@ on: branches: - master - main + - bids23 paths: # Only rebuild website when apps have changed @@ -127,9 +128,14 @@ jobs: - run: yq -i eval '(.$graph[] | select (.id == "crop") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/crop:${{needs.version.outputs.app-version}}"' water-bodies/app-package.cwl - run: yq -i eval '(.$graph[] | select (.id == "norm_diff") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/norm_diff:${{needs.version.outputs.app-version}}"' water-bodies/app-package.cwl - run: yq -i eval '(.$graph[] | select (.id == "otsu") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/otsu:${{needs.version.outputs.app-version}}"' water-bodies/app-package.cwl - - run: yq -i eval '(.$graph[] | select (.id == "stac") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/stac:${{needs.version.outputs.app-version}}"' water-bodies/app-package.cwl + - run: yq -i eval '(.$graph[] | select (.id == "stac") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/stac:${{needs.version.outputs.app-version}}"' water-bodies/app-package.cwl + - run: yq -i eval '(.$graph[] | select (.id == "crop") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/crop:${{needs.version.outputs.app-version}}"' water-bodies/app-package-cloud-native.cwl + - run: yq -i eval '(.$graph[] | select (.id == "norm_diff") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/norm_diff:${{needs.version.outputs.app-version}}"' water-bodies/app-package-cloud-native.cwl + - run: yq -i eval '(.$graph[] | select (.id == "otsu") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/otsu:${{needs.version.outputs.app-version}}"' water-bodies/app-package-cloud-native.cwl + - run: yq -i eval '(.$graph[] | select (.id == "stac") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/stac:${{needs.version.outputs.app-version}}"' water-bodies/app-package-cloud-native.cwl - run: mkdir downloads - run: cp water-bodies/app-package.cwl downloads/app-water-bodies.${{needs.version.outputs.app-version}}.cwl + - run: cp water-bodies/app-package-cloud-native.cwl downloads/app-water-bodies-cloud-native.${{needs.version.outputs.app-version}}.cwl - run: ls downloads/app-water-bodies.${{needs.version.outputs.app-version}}.cwl - uses: actions/upload-artifact@v2 with: @@ -154,4 +160,13 @@ jobs: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: downloads/app-water-bodies.${{needs.version.outputs.app-version}}.cwl asset_name: app-water-bodies.${{needs.version.outputs.app-version}}.cwl + asset_content_type: text/yaml + - name: upload linux artifact cloud native + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: downloads/app-water-bodies-cloud-native.${{needs.version.outputs.app-version}}.cwl + asset_name: app-water-bodies-cloud-native.${{needs.version.outputs.app-version}}.cwl asset_content_type: text/yaml \ No newline at end of file diff --git a/water-bodies/app-package-cloud-native.cwl b/water-bodies/app-package-cloud-native.cwl new file mode 100644 index 0000000..0e422f8 --- /dev/null +++ b/water-bodies/app-package-cloud-native.cwl @@ -0,0 +1,264 @@ +cwlVersion: v1.0 + +$namespaces: + s: https://schema.org/ +s:softwareVersion: 1.1.9 +schemas: +- http://schema.org/version/9.0/schemaorg-current-http.rdf + +$graph: +- class: Workflow + + id: water_bodies + label: Water bodies detection based on NDWI and otsu threshold + doc: Water bodies detection based on NDWI and otsu threshold + + requirements: + - class: ScatterFeatureRequirement + - class: SubworkflowFeatureRequirement + + inputs: + aoi: + label: area of interest + doc: area of interest as a bounding box + type: string + epsg: + label: EPSG code + doc: EPSG code + type: string + default: "EPSG:4326" + stac_items: + label: Sentinel-2 STAC items + doc: list of Sentinel-2 COG STAC items + type: string[] + + outputs: + - id: stac_catalog + outputSource: + - node_stac/stac_catalog + type: Directory + + steps: + node_water_bodies: + run: "#detect_water_body" + in: + item: stac_items + aoi: aoi + epsg: epsg + out: + - detected_water_body + scatter: item + scatterMethod: dotproduct + node_stac: + run: "#stac" + in: + item: stac_items + rasters: + source: node_water_bodies/detected_water_body + out: + - stac_catalog + +- class: Workflow + + id: detect_water_body + label: Water body detection based on NDWI and otsu threshold + doc: Water body detection based on NDWI and otsu threshold + + requirements: + - class: ScatterFeatureRequirement + + inputs: + aoi: + doc: area of interest as a bounding box + type: string + epsg: + doc: EPSG code + type: string + default: "EPSG:4326" + bands: + doc: bands used for the NDWI + type: string[] + default: ["green", "nir"] + item: + doc: STAC item + type: string + + outputs: + - id: detected_water_body + outputSource: + - node_otsu/binary_mask_item + type: File + + steps: + node_crop: + run: "#crop" + in: + item: item + aoi: aoi + epsg: epsg + band: + default: ["green", "nir"] + out: + - cropped + scatter: band + scatterMethod: dotproduct + node_normalized_difference: + run: "#norm_diff" + in: + rasters: + source: node_crop/cropped + out: + - ndwi + node_otsu: + run: "#otsu" + in: + raster: + source: node_normalized_difference/ndwi + out: + - binary_mask_item + +- class: CommandLineTool + id: crop + + requirements: + InlineJavascriptRequirement: {} + EnvVarRequirement: + envDef: + PATH: /srv/conda/envs/env_crop/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PYTHONPATH: /workspaces/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/crop:/home/jovyan/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/crop:/home/jovyan/water-bodies/command-line-tools/crop:/workspaces/vscode-binder/command-line-tools/crop + PROJ_LIB: /srv/conda/envs/env_crop/share/proj/ + ResourceRequirement: + coresMax: 2 + ramMax: 2028 + + hints: + DockerRequirement: + dockerPull: crop + + baseCommand: ["python", "-m", "app"] + arguments: [] + inputs: + item: + type: string + inputBinding: + prefix: --input-item + aoi: + type: string + inputBinding: + prefix: --aoi + epsg: + type: string + inputBinding: + prefix: --epsg + band: + type: string + inputBinding: + prefix: --band + outputs: + cropped: + outputBinding: + glob: '*.tif' + type: File + +- class: CommandLineTool + id: norm_diff + + requirements: + InlineJavascriptRequirement: {} + EnvVarRequirement: + envDef: + PATH: /srv/conda/envs/env_norm_diff/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PYTHONPATH: /workspaces/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/norm_diff:/home/jovyan/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/norm_diff:/workspaces/vscode-binder/command-line-tools/norm_diff + PROJ_LIB: /srv/conda/envs/env_norm_diff/share/proj/ + ResourceRequirement: + coresMax: 2 + ramMax: 2028 + + hints: + DockerRequirement: + dockerPull: norm_diff + + baseCommand: ["python", "-m", "app"] + arguments: [] + inputs: + rasters: + type: File[] + inputBinding: + position: 1 + outputs: + ndwi: + outputBinding: + glob: '*.tif' + type: File + +- class: CommandLineTool + id: otsu + + requirements: + InlineJavascriptRequirement: {} + EnvVarRequirement: + envDef: + PATH: /srv/conda/envs/env_otsu/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PYTHONPATH: /workspaces/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/otsu:/home/jovyan/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/otsu:/workspaces/vscode-binder/command-line-tools/otsu + PROJ_LIB: /srv/conda/envs/env_otsu/share/proj/ + ResourceRequirement: + coresMax: 2 + ramMax: 2028 + + hints: + DockerRequirement: + dockerPull: otsu + + baseCommand: ["python", "-m", "app"] + arguments: [] + inputs: + raster: + type: File + inputBinding: + position: 1 + outputs: + binary_mask_item: + outputBinding: + glob: '*.tif' + type: File + +- class: CommandLineTool + id: stac + + requirements: + InlineJavascriptRequirement: {} + EnvVarRequirement: + envDef: + PATH: /srv/conda/envs/env_stac/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PYTHONPATH: /workspaces/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/stac:/home/jovyan/ogc-eo-application-package-hands-on/water-bodies/command-line-tools/stac:/workspaces/vscode-binder/command-line-tools/stac + PROJ_LIB: /srv/conda/envs/env_stac/lib/python3.9/site-packages/rasterio/proj_data + ResourceRequirement: + coresMax: 2 + ramMax: 2028 + + hints: + DockerRequirement: + dockerPull: stac + + baseCommand: ["python", "-m", "app"] + arguments: [] + inputs: + item: + type: + type: array + items: string + inputBinding: + prefix: --input-item + + rasters: + type: + type: array + items: File + inputBinding: + prefix: --water-body + + outputs: + stac_catalog: + outputBinding: + glob: . + type: Directory \ No newline at end of file diff --git a/water-bodies/app-package.cwl b/water-bodies/app-package.cwl index 1a45f37..a1cbfbc 100644 --- a/water-bodies/app-package.cwl +++ b/water-bodies/app-package.cwl @@ -2,7 +2,7 @@ cwlVersion: v1.0 $namespaces: s: https://schema.org/ -s:softwareVersion: 1.1.8 +s:softwareVersion: 1.1.9 schemas: - http://schema.org/version/9.0/schemaorg-current-http.rdf @@ -30,7 +30,7 @@ $graph: stac_items: label: Sentinel-2 STAC items doc: list of Sentinel-2 COG STAC items - type: string[] + type: Directory[] outputs: - id: stac_catalog @@ -81,7 +81,7 @@ $graph: default: ["green", "nir"] item: doc: STAC item - type: string + type: Directory outputs: - id: detected_water_body @@ -139,7 +139,7 @@ $graph: arguments: [] inputs: item: - type: string + type: Directory inputBinding: prefix: --input-item aoi: diff --git a/water-bodies/command-line-tools/crop/app.py b/water-bodies/command-line-tools/crop/app.py index fc4ae28..411287c 100644 --- a/water-bodies/command-line-tools/crop/app.py +++ b/water-bodies/command-line-tools/crop/app.py @@ -1,3 +1,4 @@ +import os import click from urllib.parse import urlparse from osgeo import gdal @@ -42,6 +43,7 @@ def get_asset(item, common_name): def vsi_href(uri): parsed = urlparse(uri) + if parsed.scheme.startswith("http"): return "/vsicurl/{}".format(uri) elif parsed.scheme.startswith("file"): @@ -62,7 +64,7 @@ def vsi_href(uri): @click.option( "--input-item", "item_url", - help="STAC Item URL", + help="STAC Item URL or staged STAC catalog", required=True, ) @click.option( @@ -85,14 +87,18 @@ def vsi_href(uri): ) def crop(item_url, aoi, band, epsg): - item = pystac.read_file(item_url) + if os.path.isdir(item_url): + catalog = pystac.read_file(os.path.join(item_url, "catalog.json")) + item = next(catalog.get_items()) + else: + item = pystac.read_file(item_url) asset = get_asset(item, band) if not asset: raise ValueError(f"Common band name {band} not found in the assets") - asset_href = vsi_href(asset.href) + asset_href = vsi_href(asset.get_absolute_href()) bbox = aoi2box(aoi)