Skip to content

Commit

Permalink
Add support for EDSC ordering of file links
Browse files Browse the repository at this point in the history
For proper behavior in EDSC, links should be ordered as follows.

HTTP science file link
Other HTTP file links
S3 science file link
Other S3 file links

This commit updates the `cmrjs` package to handle the HTTP/S3 grouping
and the `update-granules-cmr-metadata-file-links` task to handle
proper placement of the science file.
  • Loading branch information
colecu committed Jun 6, 2023
1 parent 5fd5feb commit b7ed0e7
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 5 deletions.
9 changes: 5 additions & 4 deletions packages/cmrjs/src/cmr-utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,8 @@ function constructOnlineAccessUrls({
}

const urlListCalls = files.map((file) => {
const urls = [];
const distribution_urls = [];
const s3_urls = [];
if (['both', 'distribution'].includes(cmrGranuleUrlType)) {
const url = constructOnlineAccessUrl({
file,
Expand All @@ -523,7 +524,7 @@ function constructOnlineAccessUrls({
distributionBucketMap,
useDirectS3Type,
});
urls.push(url);
distribution_urls.push(url);
}
if (['both', 's3'].includes(cmrGranuleUrlType)) {
const url = constructOnlineAccessUrl({
Expand All @@ -534,9 +535,9 @@ function constructOnlineAccessUrls({
distributionBucketMap,
useDirectS3Type,
});
urls.push(url);
s3_urls.push(url);
}
return urls;
return distribution_urls.concat(s3_urls);
});
const urlList = flatten(urlListCalls);
return urlList.filter((urlObj) => urlObj);
Expand Down
24 changes: 23 additions & 1 deletion tasks/update-granules-cmr-metadata-file-links/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,26 @@ async function updateEachCmrFileAccessURLs(
}));
}

/**
* Reorder each granule's files to support EDSC's expectation of the science file occurring
* first in the file list.
*
* @param {Array<Object>} granules - array of objects that include granule information
* (e.g. { bucket: distribution path })
* @returns {Array<Object>} Array of granules with updated ordering
*/
function updateFileOrderForEdsc(granules) {
return granules.map((granule) => {
const scienceFileIndex = granule.files.findIndex((file) => file.type === 'data');

if (scienceFileIndex !== -1) {
granule.files.unshift(granule.files.splice(scienceFileIndex, 1)[0]);
}

return granule;
});
}

async function updateGranulesCmrMetadataFileLinks(event) {
const config = event.config;
const bucketsConfig = new BucketsConfig(config.buckets);
Expand All @@ -65,7 +85,9 @@ async function updateGranulesCmrMetadataFileLinks(event) {
const cmrGranuleUrlType = get(config, 'cmrGranuleUrlType', 'both');

const incomingETags = event.config.etags || {};
const granules = event.input.granules.map((g) => addEtagsToFileObjects(g, incomingETags));
let granules = event.input.granules.map((g) => addEtagsToFileObjects(g, incomingETags));
granules = updateFileOrderForEdsc(granules);

const cmrFiles = granulesToCmrFileObjects(granules);
const granulesByGranuleId = keyBy(granules, 'granuleId');

Expand Down

0 comments on commit b7ed0e7

Please sign in to comment.