Skip to content

Commit 865c4fa

Browse files
wphyojplwphyojpl
committed
Merge branch 'develop' of github.com:unity-sds/unity-data-services into develop
2 parents 68dd194 + bbd9aef commit 865c4fa

27 files changed

+532
-0
lines changed

‎cwl/stage-in-daac/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Test stage in data from DAAC
2+
3+
Update `stac_json` value in stage-in-job-01.yml
4+
5+
Run the following command with a CWL runner.
6+
7+
`cwltool stage-in-workflow.cwl stage-in-job-01.yml`

‎cwl/stage-in-daac/stage-in-job-01.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
download_dir: "granules"
2+
stac_json: "https://cmr.earthdata.nasa.gov/stac/LPCLOUD/collections/EMITL1BRAD_001/items?limit=2"

‎cwl/stage-in-daac/stage-in-workflow.cwl‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env cwl-runner
2+
cwlVersion: v1.2
3+
class: Workflow
4+
label: Workflow that executes the Sounder SIPS end-to-end chirp rebinngin workflow
5+
6+
$namespaces:
7+
cwltool: http://commonwl.org/cwltool#
8+
9+
requirements:
10+
SubworkflowFeatureRequirement: {}
11+
StepInputExpressionRequirement: {}
12+
InlineJavascriptRequirement: {}
13+
NetworkAccess:
14+
networkAccess: true
15+
16+
inputs:
17+
download_dir: string
18+
stac_json: string
19+
20+
outputs:
21+
data:
22+
type: Directory
23+
outputSource: stage-in/download_dir
24+
25+
steps:
26+
stage-in:
27+
run: "stage-in.cwl"
28+
in:
29+
stac_json: stac_json
30+
download_dir: download_dir
31+
out: [download_dir]

‎cwl/stage-in-daac/stage-in.cwl‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
cwlVersion: v1.2
2+
class: CommandLineTool
3+
4+
baseCommand: ["DOWNLOAD"]
5+
6+
requirements:
7+
DockerRequirement:
8+
dockerPull: ghcr.io/unity-sds/unity-data-services:9.4.0
9+
NetworkAccess:
10+
networkAccess: true
11+
EnvVarRequirement:
12+
envDef:
13+
DOWNLOAD_DIR: $(runtime.outdir)/$(inputs.download_dir)
14+
STAC_JSON: $(inputs.stac_json)
15+
LOG_LEVEL: '10'
16+
PARALLEL_COUNT: '-1'
17+
DOWNLOAD_RETRY_WAIT_TIME: '30'
18+
DOWNLOAD_RETRY_TIMES: '5'
19+
DOWNLOADING_ROLES: 'data'
20+
21+
EDL_BASE_URL: 'https://urs.earthdata.nasa.gov/'
22+
EDL_USERNAME: '/sps/processing/workflows/edl_username'
23+
EDL_PASSWORD: '/sps/processing/workflows/edl_password'
24+
EDL_PASSWORD_TYPE: 'PARAM_STORE'
25+
26+
VERIFY_SSL: 'TRUE'
27+
STAC_AUTH_TYPE: 'NONE'
28+
29+
inputs:
30+
download_dir:
31+
type: string
32+
stac_json:
33+
type: string
34+
35+
outputs:
36+
download_dir:
37+
type: Directory
38+
outputBinding:
39+
glob: "$(inputs.download_dir)"

‎cwl/stage-in-json-str/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Test stage in data from DAAC
2+
3+
Update `stac_json` value in stage-in-job-01.yml
4+
5+
Run the following command with a CWL runner.
6+
7+
`cwltool stage-in-workflow.cwl stage-in-job-01.yml`

‎cwl/stage-in-json-str/stage-in-job-01.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
download_dir: "granules"
2+
stac_json: '{"type":"FeatureCollection","stac_version":"1.0.0","numberMatched":2,"numberReturned":2,"features":[{"properties":{"datetime":"2016-08-22T00:05:22.000Z","start_datetime":"2016-08-22T00:05:22.000Z","end_datetime":"2016-08-22T00:11:22.000Z"},"bbox":[-7.02,-60.32,26.31,-36.16],"assets":{"data":{"title":"Download SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc","href":"https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0005.m06.g001.L1_AQ.std.v02_48.G.200425095850.nc"}},"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[6.18,-36.16],[-7.02,-56.04],[23.24,-60.32],[26.31,-38.94],[6.18,-36.16]]]},"stac_extensions":[],"id":"G2040068613-GES_DISC","stac_version":"1.0.0","collection":"C2011289787-GES_DISC","links":[{"rel":"self","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.stac"},{"rel":"parent","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"collection","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"root","href":"https://cmr.earthdata.nasa.gov:443/search/"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.json"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068613-GES_DISC.umm_json"}]},{"properties":{"datetime":"2016-08-22T00:11:22.000Z","start_datetime":"2016-08-22T00:11:22.000Z","end_datetime":"2016-08-22T00:17:22.000Z"},"bbox":[-43.78,-81.77028018298317,23.22,-56.18],"assets":{"data":{"title":"Download SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc","href":"https://raw.githubusercontent.com/unity-sds/unity-tutorial-application/main/test/stage_in/SNDR.SS1330.CHIRP.20160822T0011.m06.g002.L1_AQ.std.v02_48.G.200425095901.nc"}},"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-7.16,-56.18],[-43.78,-71.72],[20.73,-81.77],[23.22,-60.47],[-7.16,-56.18]]]},"stac_extensions":[],"id":"G2040068619-GES_DISC","stac_version":"1.0.0","collection":"C2011289787-GES_DISC","links":[{"rel":"self","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.stac"},{"rel":"parent","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"collection","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/C2011289787-GES_DISC.stac"},{"rel":"root","href":"https://cmr.earthdata.nasa.gov:443/search/"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.json"},{"rel":"via","href":"https://cmr.earthdata.nasa.gov:443/search/concepts/G2040068619-GES_DISC.umm_json"}]}],"links":[{"rel":"self","href":"https://cmr.earthdata.nasa.gov:443/search/granules.stac?collection_concept_id=C2011289787-GES_DISC&temporal%5B%5D=2016-08-22T00%3A10%3A00%2C2016-08-22T00%3A15%3A00&page_num=1"},{"rel":"root","href":"https://cmr.earthdata.nasa.gov:443/search/"}],"context":{"returned":2,"limit":1000000,"matched":2}}'

‎cwl/stage-in-json-str/stage-in-workflow.cwl‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env cwl-runner
2+
cwlVersion: v1.2
3+
class: Workflow
4+
label: Workflow that executes the Sounder SIPS end-to-end chirp rebinngin workflow
5+
6+
$namespaces:
7+
cwltool: http://commonwl.org/cwltool#
8+
9+
requirements:
10+
SubworkflowFeatureRequirement: {}
11+
StepInputExpressionRequirement: {}
12+
InlineJavascriptRequirement: {}
13+
NetworkAccess:
14+
networkAccess: true
15+
16+
inputs:
17+
download_dir: string
18+
stac_json: string
19+
20+
outputs:
21+
data:
22+
type: Directory
23+
outputSource: stage-in/download_dir
24+
25+
steps:
26+
stage-in:
27+
run: "stage-in.cwl"
28+
in:
29+
stac_json: stac_json
30+
download_dir: download_dir
31+
out: [download_dir]

‎cwl/stage-in-json-str/stage-in.cwl‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
cwlVersion: v1.2
2+
class: CommandLineTool
3+
4+
baseCommand: ["DOWNLOAD"]
5+
6+
requirements:
7+
DockerRequirement:
8+
dockerPull: ghcr.io/unity-sds/unity-data-services:9.4.0
9+
NetworkAccess:
10+
networkAccess: true
11+
EnvVarRequirement:
12+
envDef:
13+
DOWNLOAD_DIR: $(runtime.outdir)/$(inputs.download_dir)
14+
STAC_JSON: $(inputs.stac_json)
15+
16+
inputs:
17+
download_dir:
18+
type: string
19+
stac_json:
20+
type: string
21+
22+
outputs:
23+
download_dir:
24+
type: Directory
25+
outputBinding:
26+
glob: "$(inputs.download_dir)"

‎cwl/stage-in-unity/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Test stage in data from MDPS
2+
3+
Update `stac_json` and `unity_client_id` values in stage-in-job-01.yml
4+
5+
Run the following command with a CWL runner.
6+
7+
`cwltool stage-in-workflow.cwl stage-in-job-01.yml`

‎cwl/stage-in-unity/stage-in-job-01.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
download_dir: "granules"
2+
stac_json: "https://d3vc8w9zcq658.cloudfront.net/am-uds-dapa/collections/urn:nasa:unity:unity:dev:SBG-L2A_RSRFL___1/items"
3+
unity_client_id: "40c2s0ulbhp9i0fmaph3su9jch"

0 commit comments

Comments
 (0)