diff --git a/workflows/test/earthscanner-false-colour.yaml b/workflows/test/earthscanner-false-colour.yaml new file mode 100644 index 000000000..da7a7412c --- /dev/null +++ b/workflows/test/earthscanner-false-colour.yaml @@ -0,0 +1,574 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: test-earthscanner-l3c- + namespace: argo +spec: + parallelism: 50 + nodeSelector: + karpenter.sh/capacity-type: "spot" + entrypoint: main + synchronization: + semaphore: + configMapKeyRef: + name: semaphores + key: standardising + arguments: + parameters: + - name: version-argo-tasks + value: "v2" + - name: version-basemaps-cli + value: "v6.39.0-15-g3e982390" + - name: version-topo-imagery + value: "v1" + - name: source + value: "s3://linz-satellite-imagery-staging/earthscanner/2023-02-21/L3C/" + - name: include + value: ".zip$" + - name: scale + value: "None" + enum: + - "500" + - "1000" + - "2000" + - "5000" + - "10000" + - "50000" + - "None" + - name: source-epsg + value: "32760" + - name: target-epsg + value: "2193" + - name: group + value: "1" + - name: compression + value: "webp" + enum: + - "webp" + - "lzw" + - name: cutline + description: "(Optional) location of a cutline file to cut the imagery to .fgb or .geojson" + value: "" + - name: title + value: "Earthscanner L3C 0.5m (2023)" + - name: description + value: "Orthophotography within the *Region Name* region captured in the *Year*-*Year* flying season." + - name: producer + value: "Chang Guang Satellite Technology" + enum: + [ + "Unknown", + "AAM NZ", + "Aerial Surveys", + "Beca", + "Chang Guang Satellite Technology", + "European Space Agency", + "GeoSmart", + "Landpro", + "Maxar", + "NZ Aerial Mapping", + "Recon", + "SkyCan", + "Terralink International", + "UAV Mapping NZ", + ] + - name: licensor + value: "Unknown" + enum: + [ + "Unknown", + "Ashburton District Council", + "Auckland Council", + "BOPLASS", + "Bay of Plenty Regional Council", + "Buller District Council", + "Canterbury Aerial Imagery Consortium (CAI)", + "Carterton District Council", + "Central Hawke's Bay District Council", + "Central Otago District Council", + "Chang Guang Satellite Technology", + "Chatham Islands Council", + "Christchurch City Council", + "Clutha District Council", + "CoLAB", + "Department of Conservation", + "Dunedin City Council", + "Environment Canterbury", + "Environment Southland", + "Far North District Council", + "Gisborne District Council", + "Gore District Council", + "Greater Wellington Regional Council", + "Grey District Council", + "Hamilton City Council", + "Hastings District Council", + "Hauraki District Council", + "Hawke's Bay Local Authority Shared Services (HB LASS)", + "Hawke's Bay Regional Council", + "Horizons Regional Council", + "Horowhenua District Council", + "Hurunui District Council", + "Hutt City Council", + "Invercargill City Council", + "Kaikōura District Council", + "Kaipara District Council", + "Kawerau District Council", + "Kāpiti Coast District Council", + "Mackenzie District Council", + "Manawatū District Council", + "Manawatū-Whanganui LASS", + "Marlborough District Council", + "Masterton District Council", + "Matamata-Piako District Council", + "Maxar Technologies", + "Ministry of Primary Industries", + "NZ Aerial Mapping", + "Napier City Council", + "Nelson City Council", + "New Plymouth District Council", + "Northland Regional Council", + "Ōpōtiki District Council", + "Ōtorohanga District Council", + "Otago Regional Council", + "Palmerston North City Council", + "Porirua City Council", + "Queenstown-Lakes District Council", + "Rangitīkei District Council", + "Rotorua District Council", + "Ruapehu District Council", + "Selwyn District Council", + "Sinergise", + "South Taranaki District Council", + "South Waikato District Council", + "South Wairarapa District Council", + "Southland District Council", + "Stratford District Council", + "Taranaki Regional Council", + "Tararua District Council", + "Tasman District Council", + "Taupō District Council", + "Tauranga City Council", + "Terralink International", + "Thames-Coromandel District Council", + "Timaru District Council", + "Toitū Te Whenua Land Information New Zealand", + "Upper Hutt City Council", + "Waikato District Council", + "Waikato Regional Aerial Photography Service (WRAPS)", + "Waikato Regional Council", + "Waimakariri District Council", + "Waimate District Council", + "Waipā District Council", + "Wairoa District Council", + "Waitaki District Council", + "Waitomo District Council", + "Waka Kotahi", + "Wellington City Council", + "West Coast Regional Council", + "Western Bay of Plenty District Council", + "Westland District Council", + "Whakatāne District Council", + "Whanganui District Council", + "Whangārei District Council", + ] + - name: licensor-list + description: "(Optional) Separate the licensor names by a semi-colon (;). It has no effect unless a semicolon-delimited list is entered." + value: "" + - name: start-datetime + value: "2023-02-21" + - name: end-datetime + value: "2023-02-21" + - name: copy-option + value: "--no-clobber" + enum: + - "--no-clobber" + - "--force" + - "--force-no-clobber" + templateDefaults: + container: + imagePullPolicy: Always + templates: + - name: main + dag: + tasks: + - name: aws-list + template: aws-list + - name: generate-ulid + template: generate-ulid + - name: standardise-validate + template: standardise-validate + arguments: + parameters: + - name: file + value: "{{item}}" + - name: collection-id + value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" + depends: "aws-list && generate-ulid" + withParam: "{{tasks.aws-list.outputs.parameters.files}}" + - name: flatten + template: flatten + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + depends: "get-location && standardise-validate" + - name: flatten-copy + template: flatten-copy + arguments: + parameters: + - name: file + value: "{{item}}" + depends: "flatten" + withParam: "{{tasks.flatten.outputs.parameters.files}}" + - name: create-collection + template: create-collection + arguments: + parameters: + - name: collection-id + value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + depends: "flatten-copy" + - name: stac-validate + template: stac-validate + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + artifacts: + - name: stac-result + raw: + data: "{{tasks.stac-validate.outputs.result}}" + depends: "create-collection" + - name: get-location + template: get-location + - name: create-overview + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + template: create-overview + depends: "get-location && flatten-copy" + - name: create-config + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + template: create-config + depends: "get-location && create-overview" + outputs: + parameters: + - name: target + valueFrom: + parameter: "{{tasks.get-location.outputs.parameters.location}}" + - name: aws-list + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "list", + "--verbose", + "--include", + "{{=sprig.trim(workflow.parameters.include)}}", + "--group", + "{{=sprig.trim(workflow.parameters.group)}}", + "--output", + "/tmp/file_list.json", + "{{=sprig.trim(workflow.parameters.source)}}", + ] + outputs: + parameters: + - name: files + valueFrom: + path: /tmp/file_list.json + - name: generate-ulid + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + command: [python] + source: | + import ulid + with open("/tmp/ulid", "w") as f: + f.write(str(ulid.ULID())) + outputs: + parameters: + - name: ulid + valueFrom: + path: "/tmp/ulid" + - name: standardise-validate + retryStrategy: + limit: "2" + nodeSelector: + karpenter.sh/capacity-type: "spot" + inputs: + parameters: + - name: file + - name: collection-id + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 15000m + ephemeral-storage: 3Gi + volumeMounts: + - name: ephemeral + mountPath: "/tmp" + command: + - "bash" + source: | + # ensure the script dies if something goes wrong + set -e + set -o xtrace + + apt install jq wget -y + + # grab s5cmd so we can get files from s3 + wget https://github.com/peak/s5cmd/releases/download/v2.0.0/s5cmd_2.0.0_Linux-64bit.tar.gz + tar xvf *.tar.gz + + # parameters are a list of tiffs, this could be expanded into a loop if needed + SOURCE_FILE=$(echo '{{inputs.parameters.file}}' | jq '.[0]' -r) + echo "$SOURCE_FILE" + ./s5cmd cp "$SOURCE_FILE" . + + ZIP_NAME=$(basename "$SOURCE_FILE") + + ls *.zip + unzip "$ZIP_NAME" + # no need to keep the zip after its extracted + rm "$ZIP_NAME" + + ls -R JL1K* + + INPUT_TIFF=$(ls -1 JL1K*/*.tif) + TIFF_NAME=$(basename $INPUT_TIFF) + echo $INPUT_TIFF + + + # Grab the metadata with the cutlines + # ./s5cmd cp s3://linz-topographic-upload/earthscanner/EarthScanner_Tiles_Metadata_FullCapture.zip . + # unzip EarthScanner_Tiles_Metadata_FullCapture.zip + + # SENSOR_NAME=$(echo $TIFF_NAME | cut -f2 -d'_') # Get the "PMSR6" from the file name + # SCENE_NAME=$(echo $TIFF_NAME | cut -f5-7 -d'_') # get the scene id 101_0034_001 + # echo "Sensor:" $SENSOR_NAME "Scene:" $SCENE_NAME + + # PRODUCT_ID=$(ogrinfo -ro -al *.dbf | grep productId | grep $SENSOR_NAME | grep $SCENE_NAME | cut -f6- -d' ' ) + # echo "Product:" $PRODUCT_ID + + # Convert the tiff into a byte re-ordering the bands that are wrong + gdal_translate \ + -of GTiff \ + -co TILED=YES \ + -co NUM_THREADS=all_cpus \ + -co COMPRESS=zstd \ + -co BIGTIFF=YES \ + -ot byte \ + -scale 250 4095 0 255 \ + -scale 200 2500 0 255 \ + -scale 300 2100 0 255 \ + -b 4 -b 3 -b 2 \ + -colorinterp red,green,blue \ + $INPUT_TIFF $INPUT_TIFF.color.tiff + + # # tiff from the zip is no longer needed, clear it up to save space + rm $INPUT_TIFF + + # remove the black edges + nearblack -of gtiff $INPUT_TIFF.color.tiff -setalpha -o $TIFF_NAME + rm $INPUT_TIFF.color.tiff + + # ogr2ogr cutline.fgb *.dbf -where "productId = '${PRODUCT_ID}'" + + python /app/scripts/standardise_validate.py \ + --source $TIFF_NAME \ + --scale {{workflow.parameters.scale}} \ + --preset {{workflow.parameters.compression}} \ + --start-datetime "{{=sprig.trim(workflow.parameters['start-datetime'])}}" \ + --end-datetime "{{=sprig.trim(workflow.parameters['end-datetime'])}}" \ + --collection-id "{{inputs.parameters.collection-id}}" \ + --source-epsg "{{=sprig.trim(workflow.parameters['source-epsg'])}}" \ + --target-epsg "{{=sprig.trim(workflow.parameters['target-epsg'])}}" + + outputs: + artifacts: + - name: standardised_tiffs + path: /tmp/ + archive: + none: {} + - name: flatten + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "create-manifest", + "--flatten", + "--verbose", + "--include", + ".tiff?$|.json$", + "--group", + "1000", + "--group-size", + "50Gi", + "--output", + "/tmp/file_list.json", + "--target", + "{{inputs.parameters.location}}flat/", + "{{inputs.parameters.location}}", + ] + outputs: + parameters: + - name: files + valueFrom: + path: /tmp/file_list.json + - name: flatten-copy + retryStrategy: + limit: "2" + inputs: + parameters: + - name: file + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 2000m + command: [node, /app/index.js] + args: + [ + "copy", + "{{workflow.parameters.copy-option}}", + "{{inputs.parameters.file}}", + ] + - name: create-collection + retryStrategy: + limit: "2" + nodeSelector: + karpenter.sh/capacity-type: "spot" + inputs: + parameters: + - name: collection-id + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 2000m + command: + - python + - "/app/scripts/collection_from_items.py" + args: + - "--uri" + - "{{inputs.parameters.location}}flat/" + - "--collection-id" + - "{{inputs.parameters.collection-id}}" + - "--title" + - "{{=sprig.trim(workflow.parameters.title)}}" + - "--description" + - "{{=sprig.trim(workflow.parameters.description)}}" + - "--producer" + - "{{workflow.parameters.producer}}" + - "--licensor" + - "{{workflow.parameters.licensor}}" + - "--licensor-list" + - "{{=sprig.trim(workflow.parameters['licensor-list'])}}" + - "--concurrency" + - "25" + - name: stac-validate + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "stac-validate", + "--recursive", + "{{inputs.parameters.location}}flat/collection.json", + ] + - name: get-location + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node] + source: | + const fs = require('fs'); + const loc = JSON.parse(process.env['ARGO_TEMPLATE']).archiveLocation.s3; + const key = loc.key.replace('{{pod.name}}',''); + fs.writeFileSync('/tmp/location', `s3://${loc.bucket}/${key}`); + outputs: + parameters: + - name: location + valueFrom: + path: "/tmp/location" + - name: create-overview + retryStrategy: + limit: "2" + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:basemaps-cli-{{=sprig.trim(workflow.parameters['version-basemaps-cli'])}}" + resources: + requests: + cpu: 3000m + memory: 7.8Gi + command: [node, index.cjs] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "-V", + "create-overview", + "--source", + "{{inputs.parameters.location}}flat/", + "--output", + "{{inputs.parameters.location}}flat/", + ] + - name: create-config + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:basemaps-cli-{{=sprig.trim(workflow.parameters['version-basemaps-cli'])}}" + command: [node, index.cjs] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "-V", + "create-config", + "--path", + "{{inputs.parameters.location}}flat/", + "--output", + "/tmp/url", + "--title", + "{{=sprig.trim(workflow.parameters.title)}}", + "--commit", + ] + outputs: + parameters: + - name: url + valueFrom: + path: "/tmp/url" + volumes: + - name: ephemeral + emptyDir: {} diff --git a/workflows/test/earthscanner.yaml b/workflows/test/earthscanner.yaml new file mode 100644 index 000000000..c69841f78 --- /dev/null +++ b/workflows/test/earthscanner.yaml @@ -0,0 +1,574 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: test-earthscanner-l3c- + namespace: argo +spec: + parallelism: 50 + nodeSelector: + karpenter.sh/capacity-type: "spot" + entrypoint: main + synchronization: + semaphore: + configMapKeyRef: + name: semaphores + key: standardising + arguments: + parameters: + - name: version-argo-tasks + value: "v2" + - name: version-basemaps-cli + value: "v6.39.0-15-g3e982390" + - name: version-topo-imagery + value: "v1" + - name: source + value: "s3://linz-satellite-imagery-staging/earthscanner/2023-02-21/L3C/" + - name: include + value: ".zip$" + - name: scale + value: "None" + enum: + - "500" + - "1000" + - "2000" + - "5000" + - "10000" + - "50000" + - "None" + - name: source-epsg + value: "32760" + - name: target-epsg + value: "2193" + - name: group + value: "1" + - name: compression + value: "webp" + enum: + - "webp" + - "lzw" + - name: cutline + description: "(Optional) location of a cutline file to cut the imagery to .fgb or .geojson" + value: "" + - name: title + value: "Earthscanner L3C 0.5m (2023)" + - name: description + value: "Orthophotography within the *Region Name* region captured in the *Year*-*Year* flying season." + - name: producer + value: "Chang Guang Satellite Technology" + enum: + [ + "Unknown", + "AAM NZ", + "Aerial Surveys", + "Beca", + "Chang Guang Satellite Technology", + "European Space Agency", + "GeoSmart", + "Landpro", + "Maxar", + "NZ Aerial Mapping", + "Recon", + "SkyCan", + "Terralink International", + "UAV Mapping NZ", + ] + - name: licensor + value: "Unknown" + enum: + [ + "Unknown", + "Ashburton District Council", + "Auckland Council", + "BOPLASS", + "Bay of Plenty Regional Council", + "Buller District Council", + "Canterbury Aerial Imagery Consortium (CAI)", + "Carterton District Council", + "Central Hawke's Bay District Council", + "Central Otago District Council", + "Chang Guang Satellite Technology", + "Chatham Islands Council", + "Christchurch City Council", + "Clutha District Council", + "CoLAB", + "Department of Conservation", + "Dunedin City Council", + "Environment Canterbury", + "Environment Southland", + "Far North District Council", + "Gisborne District Council", + "Gore District Council", + "Greater Wellington Regional Council", + "Grey District Council", + "Hamilton City Council", + "Hastings District Council", + "Hauraki District Council", + "Hawke's Bay Local Authority Shared Services (HB LASS)", + "Hawke's Bay Regional Council", + "Horizons Regional Council", + "Horowhenua District Council", + "Hurunui District Council", + "Hutt City Council", + "Invercargill City Council", + "Kaikōura District Council", + "Kaipara District Council", + "Kawerau District Council", + "Kāpiti Coast District Council", + "Mackenzie District Council", + "Manawatū District Council", + "Manawatū-Whanganui LASS", + "Marlborough District Council", + "Masterton District Council", + "Matamata-Piako District Council", + "Maxar Technologies", + "Ministry of Primary Industries", + "NZ Aerial Mapping", + "Napier City Council", + "Nelson City Council", + "New Plymouth District Council", + "Northland Regional Council", + "Ōpōtiki District Council", + "Ōtorohanga District Council", + "Otago Regional Council", + "Palmerston North City Council", + "Porirua City Council", + "Queenstown-Lakes District Council", + "Rangitīkei District Council", + "Rotorua District Council", + "Ruapehu District Council", + "Selwyn District Council", + "Sinergise", + "South Taranaki District Council", + "South Waikato District Council", + "South Wairarapa District Council", + "Southland District Council", + "Stratford District Council", + "Taranaki Regional Council", + "Tararua District Council", + "Tasman District Council", + "Taupō District Council", + "Tauranga City Council", + "Terralink International", + "Thames-Coromandel District Council", + "Timaru District Council", + "Toitū Te Whenua Land Information New Zealand", + "Upper Hutt City Council", + "Waikato District Council", + "Waikato Regional Aerial Photography Service (WRAPS)", + "Waikato Regional Council", + "Waimakariri District Council", + "Waimate District Council", + "Waipā District Council", + "Wairoa District Council", + "Waitaki District Council", + "Waitomo District Council", + "Waka Kotahi", + "Wellington City Council", + "West Coast Regional Council", + "Western Bay of Plenty District Council", + "Westland District Council", + "Whakatāne District Council", + "Whanganui District Council", + "Whangārei District Council", + ] + - name: licensor-list + description: "(Optional) Separate the licensor names by a semi-colon (;). It has no effect unless a semicolon-delimited list is entered." + value: "" + - name: start-datetime + value: "2023-02-21" + - name: end-datetime + value: "2023-02-21" + - name: copy-option + value: "--no-clobber" + enum: + - "--no-clobber" + - "--force" + - "--force-no-clobber" + templateDefaults: + container: + imagePullPolicy: Always + templates: + - name: main + dag: + tasks: + - name: aws-list + template: aws-list + - name: generate-ulid + template: generate-ulid + - name: standardise-validate + template: standardise-validate + arguments: + parameters: + - name: file + value: "{{item}}" + - name: collection-id + value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" + depends: "aws-list && generate-ulid" + withParam: "{{tasks.aws-list.outputs.parameters.files}}" + - name: flatten + template: flatten + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + depends: "get-location && standardise-validate" + - name: flatten-copy + template: flatten-copy + arguments: + parameters: + - name: file + value: "{{item}}" + depends: "flatten" + withParam: "{{tasks.flatten.outputs.parameters.files}}" + - name: create-collection + template: create-collection + arguments: + parameters: + - name: collection-id + value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + depends: "flatten-copy" + - name: stac-validate + template: stac-validate + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + artifacts: + - name: stac-result + raw: + data: "{{tasks.stac-validate.outputs.result}}" + depends: "create-collection" + - name: get-location + template: get-location + - name: create-overview + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + template: create-overview + depends: "get-location && flatten-copy" + - name: create-config + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + template: create-config + depends: "get-location && create-overview" + outputs: + parameters: + - name: target + valueFrom: + parameter: "{{tasks.get-location.outputs.parameters.location}}" + - name: aws-list + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "list", + "--verbose", + "--include", + "{{=sprig.trim(workflow.parameters.include)}}", + "--group", + "{{=sprig.trim(workflow.parameters.group)}}", + "--output", + "/tmp/file_list.json", + "{{=sprig.trim(workflow.parameters.source)}}", + ] + outputs: + parameters: + - name: files + valueFrom: + path: /tmp/file_list.json + - name: generate-ulid + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + command: [python] + source: | + import ulid + with open("/tmp/ulid", "w") as f: + f.write(str(ulid.ULID())) + outputs: + parameters: + - name: ulid + valueFrom: + path: "/tmp/ulid" + - name: standardise-validate + retryStrategy: + limit: "2" + nodeSelector: + karpenter.sh/capacity-type: "spot" + inputs: + parameters: + - name: file + - name: collection-id + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 15000m + ephemeral-storage: 3Gi + volumeMounts: + - name: ephemeral + mountPath: "/tmp" + command: + - "bash" + source: | + # ensure the script dies if something goes wrong + set -e + set -o xtrace + + apt install jq wget -y + + # grab s5cmd so we can get files from s3 + wget https://github.com/peak/s5cmd/releases/download/v2.0.0/s5cmd_2.0.0_Linux-64bit.tar.gz + tar xvf *.tar.gz + + # parameters are a list of tiffs, this could be expanded into a loop if needed + SOURCE_FILE=$(echo '{{inputs.parameters.file}}' | jq '.[0]' -r) + echo "$SOURCE_FILE" + ./s5cmd cp "$SOURCE_FILE" . + + ZIP_NAME=$(basename "$SOURCE_FILE") + + ls *.zip + unzip "$ZIP_NAME" + # no need to keep the zip after its extracted + rm "$ZIP_NAME" + + ls -R JL1K* + + INPUT_TIFF=$(ls -1 JL1K*/*.tif) + TIFF_NAME=$(basename $INPUT_TIFF) + echo $INPUT_TIFF + + + # Grab the metadata with the cutlines + # ./s5cmd cp s3://linz-topographic-upload/earthscanner/EarthScanner_Tiles_Metadata_FullCapture.zip . + # unzip EarthScanner_Tiles_Metadata_FullCapture.zip + + # SENSOR_NAME=$(echo $TIFF_NAME | cut -f2 -d'_') # Get the "PMSR6" from the file name + # SCENE_NAME=$(echo $TIFF_NAME | cut -f5-7 -d'_') # get the scene id 101_0034_001 + # echo "Sensor:" $SENSOR_NAME "Scene:" $SCENE_NAME + + # PRODUCT_ID=$(ogrinfo -ro -al *.dbf | grep productId | grep $SENSOR_NAME | grep $SCENE_NAME | cut -f6- -d' ' ) + # echo "Product:" $PRODUCT_ID + + # Convert the tiff into a byte re-ordering the bands that are wrong + gdal_translate \ + -of GTiff \ + -co TILED=YES \ + -co NUM_THREADS=all_cpus \ + -co COMPRESS=zstd \ + -co BIGTIFF=YES \ + -ot byte \ + -scale 200 2500 0 255 \ + -scale 300 2100 0 255 \ + -scale 300 1700 0 255 \ + -b 3 -b 2 -b 1 \ + -colorinterp red,green,blue \ + $INPUT_TIFF $INPUT_TIFF.color.tiff + + # # tiff from the zip is no longer needed, clear it up to save space + rm $INPUT_TIFF + + # remove the black edges + nearblack -of gtiff $INPUT_TIFF.color.tiff -setalpha -o $TIFF_NAME + rm $INPUT_TIFF.color.tiff + + # ogr2ogr cutline.fgb *.dbf -where "productId = '${PRODUCT_ID}'" + + python /app/scripts/standardise_validate.py \ + --source $TIFF_NAME \ + --scale {{workflow.parameters.scale}} \ + --preset {{workflow.parameters.compression}} \ + --start-datetime "{{=sprig.trim(workflow.parameters['start-datetime'])}}" \ + --end-datetime "{{=sprig.trim(workflow.parameters['end-datetime'])}}" \ + --collection-id "{{inputs.parameters.collection-id}}" \ + --source-epsg "{{=sprig.trim(workflow.parameters['source-epsg'])}}" \ + --target-epsg "{{=sprig.trim(workflow.parameters['target-epsg'])}}" + + outputs: + artifacts: + - name: standardised_tiffs + path: /tmp/ + archive: + none: {} + - name: flatten + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "create-manifest", + "--flatten", + "--verbose", + "--include", + ".tiff?$|.json$", + "--group", + "1000", + "--group-size", + "50Gi", + "--output", + "/tmp/file_list.json", + "--target", + "{{inputs.parameters.location}}flat/", + "{{inputs.parameters.location}}", + ] + outputs: + parameters: + - name: files + valueFrom: + path: /tmp/file_list.json + - name: flatten-copy + retryStrategy: + limit: "2" + inputs: + parameters: + - name: file + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 2000m + command: [node, /app/index.js] + args: + [ + "copy", + "{{workflow.parameters.copy-option}}", + "{{inputs.parameters.file}}", + ] + - name: create-collection + retryStrategy: + limit: "2" + nodeSelector: + karpenter.sh/capacity-type: "spot" + inputs: + parameters: + - name: collection-id + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 2000m + command: + - python + - "/app/scripts/collection_from_items.py" + args: + - "--uri" + - "{{inputs.parameters.location}}flat/" + - "--collection-id" + - "{{inputs.parameters.collection-id}}" + - "--title" + - "{{=sprig.trim(workflow.parameters.title)}}" + - "--description" + - "{{=sprig.trim(workflow.parameters.description)}}" + - "--producer" + - "{{workflow.parameters.producer}}" + - "--licensor" + - "{{workflow.parameters.licensor}}" + - "--licensor-list" + - "{{=sprig.trim(workflow.parameters['licensor-list'])}}" + - "--concurrency" + - "25" + - name: stac-validate + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "stac-validate", + "--recursive", + "{{inputs.parameters.location}}flat/collection.json", + ] + - name: get-location + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node] + source: | + const fs = require('fs'); + const loc = JSON.parse(process.env['ARGO_TEMPLATE']).archiveLocation.s3; + const key = loc.key.replace('{{pod.name}}',''); + fs.writeFileSync('/tmp/location', `s3://${loc.bucket}/${key}`); + outputs: + parameters: + - name: location + valueFrom: + path: "/tmp/location" + - name: create-overview + retryStrategy: + limit: "2" + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:basemaps-cli-{{=sprig.trim(workflow.parameters['version-basemaps-cli'])}}" + resources: + requests: + cpu: 3000m + memory: 7.8Gi + command: [node, index.cjs] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "-V", + "create-overview", + "--source", + "{{inputs.parameters.location}}flat/", + "--output", + "{{inputs.parameters.location}}flat/", + ] + - name: create-config + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:basemaps-cli-{{=sprig.trim(workflow.parameters['version-basemaps-cli'])}}" + command: [node, index.cjs] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "-V", + "create-config", + "--path", + "{{inputs.parameters.location}}flat/", + "--output", + "/tmp/url", + "--title", + "{{=sprig.trim(workflow.parameters.title)}}", + "--commit", + ] + outputs: + parameters: + - name: url + valueFrom: + path: "/tmp/url" + volumes: + - name: ephemeral + emptyDir: {} diff --git a/workflows/test/rgbi-example.yaml b/workflows/test/rgbi-example.yaml new file mode 100644 index 000000000..57f05ea4b --- /dev/null +++ b/workflows/test/rgbi-example.yaml @@ -0,0 +1,571 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: test-earthscanner-rgbi- + namespace: argo +spec: + parallelism: 50 + nodeSelector: + karpenter.sh/capacity-type: "spot" + entrypoint: main + synchronization: + semaphore: + configMapKeyRef: + name: semaphores + key: standardising + arguments: + parameters: + - name: version-argo-tasks + value: "v2" + - name: version-basemaps-cli + value: "v6.39.0-15-g3e982390" + - name: version-topo-imagery + value: "v1" + - name: source + value: "s3://linz-satellite-imagery-staging/earthscanner/2023-02-21/L3C/JL1KF01B_PMSR3_20230221054314_200135719_101_0026_001_L3C.zip" + # only one example has been processed; waiting for feedback + # value: "s3://linz-satellite-imagery-staging/earthscanner/2023-02-21/L3C/" + - name: include + value: ".zip$" + - name: scale + value: "None" + enum: + - "500" + - "1000" + - "2000" + - "5000" + - "10000" + - "50000" + - "None" + - name: source-epsg + value: "32760" + - name: target-epsg + value: "2193" + - name: group + value: "1" + - name: compression + value: "webp" + enum: + - "webp" + - "lzw" + - name: cutline + description: "(Optional) location of a cutline file to cut the imagery to .fgb or .geojson" + value: "" + - name: title + value: "Earthscanner L3C 0.5m (2023)" + - name: description + value: "Orthophotography within the *Region Name* region captured in the *Year*-*Year* flying season." + - name: producer + value: "Chang Guang Satellite Technology" + enum: + [ + "Unknown", + "AAM NZ", + "Aerial Surveys", + "Beca", + "Chang Guang Satellite Technology", + "European Space Agency", + "GeoSmart", + "Landpro", + "Maxar", + "NZ Aerial Mapping", + "Recon", + "SkyCan", + "Terralink International", + "UAV Mapping NZ", + ] + - name: licensor + value: "Unknown" + enum: + [ + "Unknown", + "Ashburton District Council", + "Auckland Council", + "BOPLASS", + "Bay of Plenty Regional Council", + "Buller District Council", + "Canterbury Aerial Imagery Consortium (CAI)", + "Carterton District Council", + "Central Hawke's Bay District Council", + "Central Otago District Council", + "Chang Guang Satellite Technology", + "Chatham Islands Council", + "Christchurch City Council", + "Clutha District Council", + "CoLAB", + "Department of Conservation", + "Dunedin City Council", + "Environment Canterbury", + "Environment Southland", + "Far North District Council", + "Gisborne District Council", + "Gore District Council", + "Greater Wellington Regional Council", + "Grey District Council", + "Hamilton City Council", + "Hastings District Council", + "Hauraki District Council", + "Hawke's Bay Local Authority Shared Services (HB LASS)", + "Hawke's Bay Regional Council", + "Horizons Regional Council", + "Horowhenua District Council", + "Hurunui District Council", + "Hutt City Council", + "Invercargill City Council", + "Kaikōura District Council", + "Kaipara District Council", + "Kawerau District Council", + "Kāpiti Coast District Council", + "Mackenzie District Council", + "Manawatū District Council", + "Manawatū-Whanganui LASS", + "Marlborough District Council", + "Masterton District Council", + "Matamata-Piako District Council", + "Maxar Technologies", + "Ministry of Primary Industries", + "NZ Aerial Mapping", + "Napier City Council", + "Nelson City Council", + "New Plymouth District Council", + "Northland Regional Council", + "Ōpōtiki District Council", + "Ōtorohanga District Council", + "Otago Regional Council", + "Palmerston North City Council", + "Porirua City Council", + "Queenstown-Lakes District Council", + "Rangitīkei District Council", + "Rotorua District Council", + "Ruapehu District Council", + "Selwyn District Council", + "Sinergise", + "South Taranaki District Council", + "South Waikato District Council", + "South Wairarapa District Council", + "Southland District Council", + "Stratford District Council", + "Taranaki Regional Council", + "Tararua District Council", + "Tasman District Council", + "Taupō District Council", + "Tauranga City Council", + "Terralink International", + "Thames-Coromandel District Council", + "Timaru District Council", + "Toitū Te Whenua Land Information New Zealand", + "Upper Hutt City Council", + "Waikato District Council", + "Waikato Regional Aerial Photography Service (WRAPS)", + "Waikato Regional Council", + "Waimakariri District Council", + "Waimate District Council", + "Waipā District Council", + "Wairoa District Council", + "Waitaki District Council", + "Waitomo District Council", + "Waka Kotahi", + "Wellington City Council", + "West Coast Regional Council", + "Western Bay of Plenty District Council", + "Westland District Council", + "Whakatāne District Council", + "Whanganui District Council", + "Whangārei District Council", + ] + - name: licensor-list + description: "(Optional) Separate the licensor names by a semi-colon (;). It has no effect unless a semicolon-delimited list is entered." + value: "" + - name: start-datetime + value: "2023-02-21" + - name: end-datetime + value: "2023-02-21" + - name: copy-option + value: "--no-clobber" + enum: + - "--no-clobber" + - "--force" + - "--force-no-clobber" + templateDefaults: + container: + imagePullPolicy: Always + templates: + - name: main + dag: + tasks: + - name: aws-list + template: aws-list + # - name: generate-ulid + # template: generate-ulid + - name: standardise-validate + template: standardise-validate + arguments: + parameters: + - name: file + value: "{{item}}" + # - name: collection-id + # value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" + depends: "aws-list" + withParam: "{{tasks.aws-list.outputs.parameters.files}}" + - name: flatten + template: flatten + arguments: + parameters: + - name: location + value: "{{tasks.get-location.outputs.parameters.location}}" + depends: "get-location && standardise-validate" + - name: flatten-copy + template: flatten-copy + arguments: + parameters: + - name: file + value: "{{item}}" + depends: "flatten" + withParam: "{{tasks.flatten.outputs.parameters.files}}" + # - name: create-collection + # template: create-collection + # arguments: + # parameters: + # - name: collection-id + # value: "{{tasks.generate-ulid.outputs.parameters.ulid}}" + # - name: location + # value: "{{tasks.get-location.outputs.parameters.location}}" + # depends: "flatten-copy" + # - name: stac-validate + # template: stac-validate + # arguments: + # parameters: + # - name: location + # value: "{{tasks.get-location.outputs.parameters.location}}" + # artifacts: + # - name: stac-result + # raw: + # data: "{{tasks.stac-validate.outputs.result}}" + # depends: "create-collection" + - name: get-location + template: get-location + # - name: create-overview + # arguments: + # parameters: + # - name: location + # value: "{{tasks.get-location.outputs.parameters.location}}" + # template: create-overview + # depends: "get-location && flatten-copy" + # - name: create-config + # arguments: + # parameters: + # - name: location + # value: "{{tasks.get-location.outputs.parameters.location}}" + # template: create-config + # depends: "get-location && create-overview" + outputs: + parameters: + - name: target + valueFrom: + parameter: "{{tasks.get-location.outputs.parameters.location}}" + - name: aws-list + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "list", + "--verbose", + "--include", + "{{=sprig.trim(workflow.parameters.include)}}", + "--group", + "{{=sprig.trim(workflow.parameters.group)}}", + "--output", + "/tmp/file_list.json", + "{{=sprig.trim(workflow.parameters.source)}}", + ] + outputs: + parameters: + - name: files + valueFrom: + path: /tmp/file_list.json + # - name: generate-ulid + # script: + # image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + # command: [python] + # source: | + # import ulid + # with open("/tmp/ulid", "w") as f: + # f.write(str(ulid.ULID())) + # outputs: + # parameters: + # - name: ulid + # valueFrom: + # path: "/tmp/ulid" + - name: standardise-validate + retryStrategy: + limit: "2" + nodeSelector: + karpenter.sh/capacity-type: "spot" + inputs: + parameters: + - name: file + # - name: collection-id + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 15000m + ephemeral-storage: 3Gi + volumeMounts: + - name: ephemeral + mountPath: "/tmp" + command: + - "bash" + source: | + # ensure the script dies if something goes wrong + set -e + set -o xtrace + + apt install jq wget -y + + # grab s5cmd so we can get files from s3 + wget https://github.com/peak/s5cmd/releases/download/v2.0.0/s5cmd_2.0.0_Linux-64bit.tar.gz + tar xvf *.tar.gz + + # parameters are a list of tiffs, this could be expanded into a loop if needed + SOURCE_FILE=$(echo '{{inputs.parameters.file}}' | jq '.[0]' -r) + echo "$SOURCE_FILE" + ./s5cmd cp "$SOURCE_FILE" . + + ZIP_NAME=$(basename "$SOURCE_FILE") + + ls *.zip + unzip "$ZIP_NAME" + # no need to keep the zip after its extracted + rm "$ZIP_NAME" + + ls -R JL1K* + + INPUT_TIFF=$(ls -1 JL1K*/*.tif) + TIFF_NAME=$(basename $INPUT_TIFF) + echo $INPUT_TIFF + + + # Grab the metadata with the cutlines + # ./s5cmd cp s3://linz-topographic-upload/earthscanner/EarthScanner_Tiles_Metadata_FullCapture.zip . + # unzip EarthScanner_Tiles_Metadata_FullCapture.zip + + # SENSOR_NAME=$(echo $TIFF_NAME | cut -f2 -d'_') # Get the "PMSR6" from the file name + # SCENE_NAME=$(echo $TIFF_NAME | cut -f5-7 -d'_') # get the scene id 101_0034_001 + # echo "Sensor:" $SENSOR_NAME "Scene:" $SCENE_NAME + + # PRODUCT_ID=$(ogrinfo -ro -al *.dbf | grep productId | grep $SENSOR_NAME | grep $SCENE_NAME | cut -f6- -d' ' ) + # echo "Product:" $PRODUCT_ID + + # Convert the tiff into a byte re-ordering the bands that are wrong + gdal_translate \ + -of GTiff \ + -co TILED=YES \ + -co NUM_THREADS=all_cpus \ + -co COMPRESS=zstd \ + -co BIGTIFF=YES \ + -b 3 -b 2 -b 1 -b 4 \ + -colorinterp red,green,blue,undefined \ + $INPUT_TIFF $INPUT_TIFF.color.tiff + + # # tiff from the zip is no longer needed, clear it up to save space + mv $INPUT_TIFF.color.tiff /tmp/ + + # remove the black edges + # nearblack -of gtiff $INPUT_TIFF.color.tiff -setalpha -o $TIFF_NAME + # rm $INPUT_TIFF.color.tiff + + # ogr2ogr cutline.fgb *.dbf -where "productId = '${PRODUCT_ID}'" + + # python /app/scripts/standardise_validate.py \ + # --source $TIFF_NAME \ + # --scale {{workflow.parameters.scale}} \ + # --preset {{workflow.parameters.compression}} \ + # --start-datetime "{{=sprig.trim(workflow.parameters['start-datetime'])}}" \ + # --end-datetime "{{=sprig.trim(workflow.parameters['end-datetime'])}}" \ + # --source-epsg "{{=sprig.trim(workflow.parameters['source-epsg'])}}" \ + # --target-epsg "{{=sprig.trim(workflow.parameters['target-epsg'])}}" + + outputs: + artifacts: + - name: standardised_tiffs + path: /tmp/ + archive: + none: {} + - name: flatten + inputs: + parameters: + - name: location + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node, /app/index.js] + env: + - name: AWS_ROLE_CONFIG_PATH + value: s3://linz-bucket-config/config.json + args: + [ + "create-manifest", + "--flatten", + "--verbose", + "--include", + ".tiff?$|.json$", + "--group", + "1000", + "--group-size", + "50Gi", + "--output", + "/tmp/file_list.json", + "--target", + "{{inputs.parameters.location}}flat/", + "{{inputs.parameters.location}}", + ] + outputs: + parameters: + - name: files + valueFrom: + path: /tmp/file_list.json + - name: flatten-copy + retryStrategy: + limit: "2" + inputs: + parameters: + - name: file + container: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + resources: + requests: + memory: 7.8Gi + cpu: 2000m + command: [node, /app/index.js] + args: + [ + "copy", + "{{workflow.parameters.copy-option}}", + "{{inputs.parameters.file}}", + ] + # - name: create-collection + # retryStrategy: + # limit: "2" + # nodeSelector: + # karpenter.sh/capacity-type: "spot" + # inputs: + # parameters: + # - name: collection-id + # - name: location + # container: + # image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:topo-imagery-{{=sprig.trim(workflow.parameters['version-topo-imagery'])}}" + # resources: + # requests: + # memory: 7.8Gi + # cpu: 2000m + # command: + # - python + # - "/app/scripts/collection_from_items.py" + # args: + # - "--uri" + # - "{{inputs.parameters.location}}flat/" + # - "--collection-id" + # - "{{inputs.parameters.collection-id}}" + # - "--title" + # - "{{=sprig.trim(workflow.parameters.title)}}" + # - "--description" + # - "{{=sprig.trim(workflow.parameters.description)}}" + # - "--producer" + # - "{{workflow.parameters.producer}}" + # - "--licensor" + # - "{{workflow.parameters.licensor}}" + # - "--licensor-list" + # - "{{=sprig.trim(workflow.parameters['licensor-list'])}}" + # - "--concurrency" + # - "25" + # - name: stac-validate + # inputs: + # parameters: + # - name: location + # container: + # image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + # command: [node, /app/index.js] + # env: + # - name: AWS_ROLE_CONFIG_PATH + # value: s3://linz-bucket-config/config.json + # args: + # [ + # "stac-validate", + # "--recursive", + # "{{inputs.parameters.location}}flat/collection.json", + # ] + - name: get-location + script: + image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:argo-tasks-{{=sprig.trim(workflow.parameters['version-argo-tasks'])}}" + command: [node] + source: | + const fs = require('fs'); + const loc = JSON.parse(process.env['ARGO_TEMPLATE']).archiveLocation.s3; + const key = loc.key.replace('{{pod.name}}',''); + fs.writeFileSync('/tmp/location', `s3://${loc.bucket}/${key}`); + outputs: + parameters: + - name: location + valueFrom: + path: "/tmp/location" + # - name: create-overview + # retryStrategy: + # limit: "2" + # inputs: + # parameters: + # - name: location + # container: + # image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:basemaps-cli-{{=sprig.trim(workflow.parameters['version-basemaps-cli'])}}" + # resources: + # requests: + # cpu: 3000m + # memory: 7.8Gi + # command: [node, index.cjs] + # env: + # - name: AWS_ROLE_CONFIG_PATH + # value: s3://linz-bucket-config/config.json + # args: + # [ + # "-V", + # "create-overview", + # "--source", + # "{{inputs.parameters.location}}flat/", + # "--output", + # "{{inputs.parameters.location}}flat/", + # ] + # - name: create-config + # inputs: + # parameters: + # - name: location + # container: + # image: "019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:basemaps-cli-{{=sprig.trim(workflow.parameters['version-basemaps-cli'])}}" + # command: [node, index.cjs] + # env: + # - name: AWS_ROLE_CONFIG_PATH + # value: s3://linz-bucket-config/config.json + # args: + # [ + # "-V", + # "create-config", + # "--path", + # "{{inputs.parameters.location}}flat/", + # "--output", + # "/tmp/url", + # "--title", + # "{{=sprig.trim(workflow.parameters.title)}}", + # "--commit", + # ] + # outputs: + # parameters: + # - name: url + # valueFrom: + # path: "/tmp/url" + volumes: + - name: ephemeral + emptyDir: {} diff --git a/workflows/test/sleep.yml b/workflows/test/sleep.yml index d04cda788..1c804e63e 100644 --- a/workflows/test/sleep.yml +++ b/workflows/test/sleep.yml @@ -6,12 +6,12 @@ spec: entrypoint: sleep templates: - name: sleep - nodeSelector: - karpenter.sh/capacity-type: "spot" + # nodeSelector: + # karpenter.sh/capacity-type: "spot" container: - resources: - requests: - memory: 3.9Gi - cpu: 2000m + # resources: + # requests: + # memory: 3.9Gi + # cpu: 2000m image: ubuntu:22.04 command: ["sleep", "3600"]