[CUMULUS-3368]: [User Contribution] Introducing allFilesPresent metadata field for Cumulus Collections. Fixes Issue #3193

CHANGELOG.md

@@ -48,6 +48,11 @@ Users/clients that do not make use of these endpoints will not be impacted.
   - Added optional `maxDownloadTime` field to `provider` schema
   - Added `max_download_time` column to PostgreSQL `providers` table
   - Updated `@cumulus/ingest/lock` to check expired locks based on `provider.maxDownloadTime`
+- **CUMULUS-3368**
+  - Added `allFilesPresent` field for `collection.meta` that defines whether or not all files
+    should be present in a granule before it can be discovered. If `allFilesPresent` is true,
+    the granules missing files will be removed. Otherwise, the default behavior is discover-granules
+    will ignore missing files in granules.
 
 ### Changed
 

docs/workflow_tasks/discover_granules.md

@@ -48,6 +48,10 @@ A list of buckets with types that will be used to assign bucket targets based on
 
 A Cumulus [collection](https://github.com/nasa/cumulus/blob/master/packages/api/lib/schemas.js) object.    Used to define granule file groupings and granule metadata for discovered files.   The collection object utilizes the collection type key to generate types in the output object on discovery.
 
+##### Collection Meta
+
+If the collection is configured with `collection.meta.allFilesPresent` set to `true`, the task will remove granules's missing files. Otherwise, the default behavior ignores missing files in granules.
+
 #### DuplicateGranuleHandling
 
 A string configuration that configures the step to filter the granules discovered:

packages/api/lib/schemas.js

@@ -216,6 +216,10 @@ module.exports.collection = {
         description: 'chunk size of the S3 multipart uploads for the collection',
         type: 'number',
       },
+      allFilesPresent: {
+        description: 'boolean flag used for DiscoverGranules task. If the collection is configured with `allFilesPresent` set to `true`, the task will remove granules\'s missing files. Otherwise, the default behavior ignores missing files in granules.',
+        type: 'boolean',
+      },
     },
     tags: {
       title: 'Optional tags for search',

packages/es-client/config/mappings.json

@@ -31,6 +31,10 @@
       "timestamp": {
         "type": "date"
       }
+    },
+    "meta": {
+      "type": "object",
+      "dynamic": false
     }
   },
   "asyncOperation": {

tasks/discover-granules/index.js

@@ -25,14 +25,14 @@ const logger = (logOptions) => new Logger({
 /**
  * Fetch a list of files from the provider
  *
- * @param {Object} params
- * @param {Object} params.providerConfig - the connection config for the provider
+ * @param {object} params
+ * @param {object} params.providerConfig - the connection config for the provider
  * @param {bool}   params.useList - flag to tell ftp server to use 'LIST'
  *   instead  of 'STAT'
  * @param {number} [params.httpRequestTimeout=300] - seconds for http provider
  *   to wait before timing out
  * @param {string} params.path - the provider path to search
- * @returns {Array<Object>} a list of discovered file objects
+ * @returns {Array<object>} a list of discovered file objects
  */
 const listFiles = async (params) => {
   const { providerConfig, useList, httpRequestTimeout = 300, path } = params;
@@ -56,7 +56,7 @@ const listFiles = async (params) => {
  *
  * @param {RegExp} granuleIdRegex - a regular expression where the first
  * matching group is the granule id
- * @param {Object} file - a file containing a `name` property
+ * @param {object} file - a file containing a `name` property
  * @returns {string|null} returns the granule id, if one could be extracted,
  * or null otherwise
  */
@@ -76,8 +76,8 @@ const granuleIdOfFile = curry(
  *
  * @param {RegExp} granuleIdRegex - a regular expression where the first
  * matching group is the granule id
- * @param {Array<Object>} files - a list of files containing a `name` property
- * @returns {Object<Array>} the files, grouped by granule id
+ * @param {Array<object>} files - a list of files containing a `name` property
+ * @returns {object<Array>} the files, grouped by granule id
  */
 const groupFilesByGranuleId = (granuleIdRegex, files) => {
   const result = groupBy(files, granuleIdOfFile(granuleIdRegex));
@@ -88,10 +88,10 @@ const groupFilesByGranuleId = (granuleIdRegex, files) => {
 /**
  * Find the collection file config associated with the file
  *
- * @param {Array<Object>} collectionFileConfigs - a list of collection file
+ * @param {Array<object>} collectionFileConfigs - a list of collection file
  * configs
- * @param {Object} file - a file
- * @returns {Object|undefined} returns the matching collection file config, or
+ * @param {object} file - a file
+ * @returns {object|undefined} returns the matching collection file config, or
  * `undefined` if a matching config could not be found
  */
 const getCollectionFileConfig = (collectionFileConfigs, file) =>
@@ -100,9 +100,9 @@ const getCollectionFileConfig = (collectionFileConfigs, file) =>
 /**
  * Check to see if a file has an associated collection file config
  *
- * @param {Array<Object>} collectionFileConfigs - a list of collection file
+ * @param {Array<object>} collectionFileConfigs - a list of collection file
  * configs
- * @param {Object} file - a file
+ * @param {object} file - a file
  * @returns {boolean}
  */
 const fileHasCollectionFileConfig = curry(
@@ -137,10 +137,10 @@ const returnAllFiles = (config) => {
  * with the file. If one is found, add `bucket`, `url_path`, and `type`
  * properties to the file.
  *
- * @param {Object} config - a config object containing `buckets` and
+ * @param {object} config - a config object containing `buckets` and
  * `collection` properties
- * @param {Object} file - a file object
- * @returns {Object} a file object, possibly with three additional properties
+ * @param {object} file - a file object
+ * @returns {object} a file object, possibly with three additional properties
  */
 const updateFileFromCollectionFileConfig = curry(
   ({ buckets, collection }, file) => {
@@ -225,6 +225,64 @@ const checkGranuleHasNoDuplicate = async (granuleId, duplicateHandling) => {
   throw new Error(`Unexpected return from Private API lambda: ${JSON.stringify(response)}`);
 };
 
+/**
+ * Checks if all files from a collection are present for files from a granuleId
+ *
+ *
+ * @param {object} config - the event config
+ * @param {object} filesByGranuleId - Object with granuleId for keys with an array of
+ *                                    matching files for each
+ * @param {string} granuleId - the Id of a granule
+ * @returns {boolean} - returns true if all file in collection config are present for granuleId
+ */
+const checkGranuleHasAllFiles = (config, filesByGranuleId, granuleId) => {
+  const granuleHasAllFiles = filesByGranuleId[granuleId].length >= config.collection.files.length;
+  return granuleHasAllFiles;
+};
+
+/**
+ * Filters out granules that are missing files from a collection
+ *
+ *
+ * @param {Object} params.filesByGranuleId - Object with granuleId for keys with an array of
+ *                                           matching files for each
+ * @param {Object} params.config - the event config
+ * @returns {Array.string} - returns granuleIds that contain all files from a collection
+ */
+const filterGranulesWithoutAllFiles = ({ filesByGranuleId, config }) => {
+  const checkResults = Object.keys(filesByGranuleId).filter(
+    checkGranuleHasAllFiles.bind(this, config, filesByGranuleId)
+  );
+  return checkResults;
+};
+
+/**
+ * Handles granules without all files present according to the allFilesPresent boolean
+ *
+ * allFilesPresent = true : granules missing files will be removed
+ * allFilesPresent = false: ignore missing files in granules
+ *
+ * @param {object} params.filesByGranuleId - Object with granuleId for keys with an array of
+ *                                           matching files for each
+ * @param {object} params.config - the event config
+ * @returns {object} returns filesByGranuleId with all the granules missing files removed
+ */
+const handleGranulesWithoutAllFilesPresent = ({ filesByGranuleId, config }) => {
+  let allFilesPresent = false;
+  if (config.collection.meta) {
+    allFilesPresent = config.collection.meta.allFilesPresent || false;
+  }
+  if (!allFilesPresent) {
+    return filesByGranuleId;
+  }
+  const filteredKeys = filterGranulesWithoutAllFiles({
+    granuleIds: Object.keys(filesByGranuleId),
+    filesByGranuleId,
+    config,
+  });
+  return pick(filesByGranuleId, filteredKeys);
+};
+
 /**
  * Filters granule duplicates from a list of granuleIds according to the
  * configuration in duplicateHandling:
@@ -258,16 +316,16 @@ const filterDuplicates = async ({ granuleIds, duplicateHandling, concurrency })
  * error:              Duplicates encountered will result in a thrown error
  * replace, version:   Duplicates will be ignored
  *
- * @param {Object} params - params object
- * @param {Object} params.filesByGranuleId - Object with granuleId for keys with an array of
+ * @param {object} params - params object
+ * @param {object} params.filesByGranuleId - Object with granuleId for keys with an array of
  *                                    matching files for each
  *
  * @param {string} params.duplicateHandling - flag that defines this function's behavior
  *                                            (see description)
  * @param {number} params.concurrency - granule duplicate filtering max concurrency
  *                                      (`skip` or `error` handling only)
  *
- * @returns {Object} returns filesByGranuleId with applicable duplciates removed
+ * @returns {object} returns filesByGranuleId with applicable duplciates removed
  */
 const handleDuplicates = async ({ filesByGranuleId, duplicateHandling, concurrency }) => {
   logger().info(`Running discoverGranules with duplicateHandling set to ${duplicateHandling}`);
@@ -314,6 +372,11 @@ const discoverGranules = async ({ config }) => {
     concurrency: get(config, 'concurrency', 3),
   });
 
+  filesByGranuleId = handleGranulesWithoutAllFilesPresent({
+    filesByGranuleId,
+    config,
+  });
+
   const discoveredGranules = map(filesByGranuleId, buildGranule(config));
 
   logger({ granules: discoveredGranules.map((g) => g.granuleId) }).info(`Discovered ${discoveredGranules.length} granules.`);
@@ -323,9 +386,9 @@ const discoverGranules = async ({ config }) => {
 /**
  * Lambda handler.
  *
- * @param {Object} event      - a Cumulus Message
- * @param {Object} context    - an AWS Lambda context
- * @returns {Promise<Object>} - Returns output from task.
+ * @param {object} event      - a Cumulus Message
+ * @param {object} context    - an AWS Lambda context
+ * @returns {Promise<object>} - Returns output from task.
  *                              See schemas/output.json for detailed output schema
  */
 const handler = async (event, context) => await runCumulusTask(discoverGranules, event, context);

tasks/discover-granules/tests/index.js

@@ -190,8 +190,8 @@ const discoverGranulesUsingS3 = (configure, assert = assertDiscoveredGranules) =
     // State sample files
     const files = [
       'granule-1.nc', 'granule-1.nc.md5',
-      'granule-2.nc', 'granule-2.nc.md5',
-      'granule-3.nc', 'granule-3.nc.md5',
+      'granule-2.nc', 'granule-2.nc.md5', 'granule-2.nc.cmr.json',
+      'granule-3.nc', 'granule-3.nc.md5', 'granule-3.nc.cmr.json', 'granule-3.nc.xml',
     ];
 
     config.sourceBucketName = randomString();
@@ -216,6 +216,126 @@ const discoverGranulesUsingS3 = (configure, assert = assertDiscoveredGranules) =
     }
   };
 
+test('discover granules with allFilesPresent set to true',
+  discoverGranulesUsingS3(({ context: { event: { config } } }) => {
+    config.provider = {
+      id: 'MODAPS',
+      protocol: 's3',
+      host: config.sourceBucketName,
+    };
+    // Adds cmr.json files to collection config
+    config.collection.granuleIdExtraction = '^(.*)\\.(nc|nc\\.md5|nc\\.cmr\\.json)$';
+    config.collection.meta = {
+      allFilesPresent: true,
+    };
+    config.collection.files = [
+      {
+        regex: '.*.nc.cmr.json$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.cmr.json',
+        bucket: 'protected',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc',
+        bucket: 'protected',
+        type: 'data',
+      },
+      {
+        regex: '.*.nc.md5$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+    ];
+  }, async (t, output) => {
+    await validateOutput(t, output);
+    t.is(output.granules.length, 2);
+    output.granules.forEach(({ files }) => t.is(files.length, 3));
+  }));
+
+test('discover granules task discovers no granules when allFilesPresent is set to true with files missing',
+  discoverGranulesUsingS3(({ context: { event: { config } } }) => {
+    config.provider = {
+      id: 'MODAPS',
+      protocol: 's3',
+      host: config.sourceBucketName,
+    };
+    // Adds cmr.json files to collection config
+    config.collection.granuleIdExtraction = '^(.*)\\.(nc|nc\\.md5|nc\\.cmr\\.json|nc\\.png)$';
+    config.collection.meta = {
+      allFilesPresent: true,
+    };
+    config.collection.files = [
+      {
+        regex: '.*.nc.cmr.json$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.cmr.json',
+        bucket: 'protected',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc',
+        bucket: 'protected',
+        type: 'data',
+      },
+      {
+        regex: '.*.nc.md5$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc.png$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+    ];
+  }, async (t, output) => {
+    await validateOutput(t, output);
+    t.is(output.granules.length, 0);
+    output.granules.forEach(({ files }) => t.is(files.length, 0));
+  }));
+
+test('discover granules task discovers granules with allFilesPresent set to true with extra files for granule',
+  discoverGranulesUsingS3(({ context: { event: { config } } }) => {
+    config.provider = {
+      id: 'MODAPS',
+      protocol: 's3',
+      host: config.sourceBucketName,
+    };
+    // Adds cmr.json and .xml granuleId
+    config.collection.granuleIdExtraction = '^(.*)\\.(nc|nc\\.md5|nc\\.cmr\\.json|nc\\.xml)$';
+    config.collection.meta = {
+      allFilesPresent: true,
+    };
+    config.collection.files = [
+      {
+        regex: '.*.nc.cmr.json$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.cmr.json',
+        bucket: 'protected',
+        type: 'metadata',
+      },
+      {
+        regex: '.*.nc$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc',
+        bucket: 'protected',
+        type: 'data',
+      },
+      {
+        regex: '.*.nc.md5$',
+        sampleFileName: '20170603090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.md5',
+        bucket: 'public',
+        type: 'metadata',
+      },
+    ];
+  }, async (t, output) => {
+    await validateOutput(t, output);
+    t.is(output.granules.length, 2);
+    output.granules.forEach(({ files }) => t.is(files.length, 3));
+  }));
+
 test('discover granules using S3',
   discoverGranulesUsingS3(({ context: { event: { config } } }) => {
     config.provider = {