CUMULUS-3694: Update granules List endpoints to query postgres - filter by field value

CHANGELOG.md

@@ -9,8 +9,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 ### Replace ElasticSearch Phase 1
 
 - **CUMULUS-3692**
-  - Update granules List endpoints to query postgres for basic queries
-
+  - Added `@cumulus/db/src/search` `BaseSearch` and `GranuleSearch` classes to
+    support basic queries for granules
+  - Updated granules List endpoint to query postgres for basic queries
+- **CUMULUS-3694**
+  - Added functionality to `@cumulus/db/src/search` to support term queries
+  - Updated `BaseSearch` and `GranuleSearch` classes to support term queries for granules
+  - Updated granules List endpoint to search postgres
 
 ### Migration Notes
 

example/spec/helpers/granuleUtils.js

@@ -234,6 +234,7 @@ const waitForGranuleRecordUpdatedInList = async (stackName, granule, additionalQ
       'beginningDateTime',
       'endingDateTime',
       'error',
+      'execution', // TODO remove after CUMULUS-3698
       'files', // TODO -2714 this should be removed
       'lastUpdateDateTime',
       'productionDateTime',

example/spec/parallel/testAPI/granuleSpec.js

@@ -183,7 +183,8 @@ describe('The Granules API', () => {
       });
 
       const searchedGranule = JSON.parse(searchResults.body).results[0];
-      expect(searchedGranule).toEqual(jasmine.objectContaining(randomGranuleRecord));
+      // TODO CUMULUS-3698 includes files
+      expect(searchedGranule).toEqual(jasmine.objectContaining(omit(randomGranuleRecord, 'files')));
     });
 
     it('can modify the granule via API.', async () => {

packages/api/endpoints/granules.js

@@ -32,7 +32,6 @@ const {
   recordNotFoundString,
   multipleRecordFoundString,
 } = require('@cumulus/es-client/search');
-const ESSearchAfter = require('@cumulus/es-client/esSearchAfter');
 
 const { deleteGranuleAndFiles } = require('../src/lib/granule-delete');
 const { zodParser } = require('../src/zod-utils');
@@ -105,25 +104,9 @@ async function list(req, res) {
   log.trace(`list query ${JSON.stringify(req.query)}`);
   const { getRecoveryStatus, ...queryStringParameters } = req.query;
 
-  let es;
-  if (queryStringParameters.searchContext) {
-    es = new ESSearchAfter(
-      { queryStringParameters },
-      'granule',
-      process.env.ES_INDEX
-    );
-  } else {
-    es = new Search({ queryStringParameters }, 'granule', process.env.ES_INDEX);
-  }
-  let result;
-  // TODO the condition should be removed after we support all the query parameters
-  if (Object.keys(queryStringParameters).filter((item) => !['limit', 'page', 'sort_key'].includes(item)).length === 0) {
-    log.debug('list perform db search');
-    const dbSearch = new GranuleSearch({ queryStringParameters });
-    result = await dbSearch.query();
-  } else {
-    result = await es.query();
-  }
+  const dbSearch = new GranuleSearch({ queryStringParameters });
+  const result = await dbSearch.query();
+
   if (getRecoveryStatus === 'true') {
     return res.send(await addOrcaRecoveryStatus(result));
   }

packages/api/tests/endpoints/test-granules.js

@@ -288,6 +288,7 @@ test.beforeEach(async (t) => {
   const granuleId1 = t.context.createGranuleId();
   const granuleId2 = t.context.createGranuleId();
   const granuleId3 = t.context.createGranuleId();
+  const timestamp = new Date();
 
   // create fake Postgres granule records
   t.context.fakePGGranules = [
@@ -299,29 +300,33 @@ test.beforeEach(async (t) => {
       cmr_link:
         'https://cmr.uat.earthdata.nasa.gov/search/granules.json?concept_id=A123456789-TEST_A',
       duration: 47.125,
-      timestamp: new Date(Date.now()),
+      timestamp,
+      updated_at: timestamp,
     }),
     fakeGranuleRecordFactory({
       granule_id: granuleId2,
       status: 'failed',
       collection_cumulus_id: t.context.collectionCumulusId,
       duration: 52.235,
-      timestamp: new Date(Date.now()),
+      timestamp,
+      updated_at: timestamp,
     }),
     fakeGranuleRecordFactory({
       granule_id: granuleId3,
       status: 'failed',
       collection_cumulus_id: t.context.collectionCumulusId,
       duration: 52.235,
-      timestamp: new Date(Date.now()),
+      timestamp,
+      updated_at: timestamp,
     }),
     // granule with same granule_id as above but different collection_cumulus_id
     fakeGranuleRecordFactory({
       granule_id: granuleId3,
       status: 'failed',
       collection_cumulus_id: t.context.collectionCumulusId2,
       duration: 52.235,
-      timestamp: new Date(Date.now()),
+      timestamp,
+      updated_at: timestamp,
     }),
   ];
 
@@ -456,7 +461,7 @@ test.serial('default lists and paginates correctly from querying database', asyn
   const { meta, results } = response.body;
   t.is(results.length, 4);
   t.is(meta.stack, process.env.stackName);
-  t.is(meta.table, 'granule');
+  t.is(meta.table, 'granules');
   t.is(meta.count, 4);
   results.forEach((r) => {
     t.true(granuleIds.includes(r.granuleId));
@@ -487,6 +492,41 @@ test.serial('default lists and paginates correctly from querying database', asyn
   t.not(results[0].granuleId, newResults[0].granuleId);
 });
 
+test.serial('LIST endpoint returns search result correctly', async (t) => {
+  const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id);
+  const searchParams = new URLSearchParams({
+    granuleId: granuleIds[3],
+  });
+  const response = await request(app)
+    .get(`/granules?limit=1&page=2&${searchParams}`)
+    .set('Accept', 'application/json')
+    .set('Authorization', `Bearer ${jwtAuthToken}`)
+    .expect(200);
+
+  const { meta, results } = response.body;
+  t.is(meta.count, 2);
+  t.is(results.length, 1);
+  t.true([granuleIds[2], granuleIds[3]].includes(results[0].granuleId));
+
+  const newSearchParams = new URLSearchParams({
+    collectionId: t.context.collectionId,
+    status: 'failed',
+    duration: 52.235,
+    timestamp: t.context.fakePGGranules[0].timestamp.getTime(),
+  });
+  const newResponse = await request(app)
+    .get(`/granules?${newSearchParams}`)
+    .set('Accept', 'application/json')
+    .set('Authorization', `Bearer ${jwtAuthToken}`)
+    .expect(200);
+
+  const { meta: newMeta, results: newResults } = newResponse.body;
+  t.is(newMeta.count, 2);
+  t.is(newResults.length, 2);
+  const newResultIds = newResults.map((g) => g.granuleId);
+  t.deepEqual([granuleIds[1], granuleIds[2]].sort(), newResultIds.sort());
+});
+
 test.serial('CUMULUS-911 GET without pathParameters and without an Authorization header returns an Authorization Missing response', async (t) => {
   const response = await request(app)
     .get('/granules')

packages/db/src/search/BaseSearch.ts

@@ -1,8 +1,11 @@
 import { Knex } from 'knex';
 import Logger from '@cumulus/logger';
-import { getKnexClient } from '../connection';
+
 import { BaseRecord } from '../types/base';
+import { getKnexClient } from '../connection';
+import { TableNames } from '../tables';
 import { DbQueryParameters, QueryEvent, QueryStringParameters } from '../types/search';
+import { convertQueryStringToDbQueryParameters } from './queries';
 
 const log = new Logger({ sender: '@cumulus/db/BaseSearch' });
 
@@ -15,32 +18,35 @@ export type Meta = {
   count?: number,
 };
 
+const typeToTable: { [key: string]: string } = {
+  asyncOperation: TableNames.asyncOperations,
+  collection: TableNames.collections,
+  execution: TableNames.executions,
+  granule: TableNames.granules,
+  pdr: TableNames.pdrs,
+  provider: TableNames.providers,
+  rule: TableNames.rules,
+};
+
 /**
  * Class to build and execute db search query
  */
 class BaseSearch {
-  readonly type?: string;
+  readonly type: string;
   readonly queryStringParameters: QueryStringParameters;
   // parsed from queryStringParameters for query build
   dbQueryParameters: DbQueryParameters = {};
 
-  constructor(event: QueryEvent, type?: string) {
+  constructor(event: QueryEvent, type: string) {
     this.type = type;
     this.queryStringParameters = event?.queryStringParameters ?? {};
-    this.dbQueryParameters.page = Number.parseInt(
-      (this.queryStringParameters.page) ?? '1',
-      10
-    );
-    this.dbQueryParameters.limit = Number.parseInt(
-      (this.queryStringParameters.limit) ?? '10',
-      10
+    this.dbQueryParameters = convertQueryStringToDbQueryParameters(
+      this.type, this.queryStringParameters
     );
-    this.dbQueryParameters.offset = (this.dbQueryParameters.page - 1)
-      * this.dbQueryParameters.limit;
   }
 
   /**
-   * build the search query
+   * Build the search query
    *
    * @param knex - DB client
    * @returns queries for getting count and search result
@@ -51,27 +57,32 @@ class BaseSearch {
       searchQuery: Knex.QueryBuilder,
     } {
     const { countQuery, searchQuery } = this.buildBasicQuery(knex);
-    if (this.dbQueryParameters.limit) searchQuery.limit(this.dbQueryParameters.limit);
-    if (this.dbQueryParameters.offset) searchQuery.offset(this.dbQueryParameters.offset);
+    this.buildTermQuery({ countQuery, searchQuery });
+    this.buildInfixPrefixQuery({ countQuery, searchQuery });
 
+    const { limit, offset } = this.dbQueryParameters;
+    if (limit) searchQuery.limit(limit);
+    if (offset) searchQuery.offset(offset);
+
+    log.debug(`_buildSearch returns countQuery: ${countQuery.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`);
     return { countQuery, searchQuery };
   }
 
   /**
-   * metadata template for query result
+   * Get metadata template for query result
    *
    * @returns metadata template
    */
   private _metaTemplate(): Meta {
     return {
       name: 'cumulus-api',
       stack: process.env.stackName,
-      table: this.type,
+      table: this.type && typeToTable[this.type],
     };
   }
 
   /**
-   * build basic query
+   * Build basic query
    *
    * @param knex - DB client
    * @throws - function is not implemented
@@ -84,6 +95,46 @@ class BaseSearch {
     throw new Error('buildBasicQuery is not implemented');
   }
 
+  /**
+   * Build queries for infix and prefix
+   *
+   * @param params
+   * @param params.countQuery - query builder for getting count
+   * @param params.searchQuery - query builder for search
+   * @param [params.dbQueryParameters] - db query parameters
+   */
+  protected buildInfixPrefixQuery(params: {
+    countQuery: Knex.QueryBuilder,
+    searchQuery: Knex.QueryBuilder,
+    dbQueryParameters?: DbQueryParameters,
+  }) {
+    log.debug(`buildInfixPrefixQuery is not implemented ${Object.keys(params)}`);
+    throw new Error('buildInfixPrefixQuery is not implemented');
+  }
+
+  /**
+   * Build queries for term fields
+   *
+   * @param params
+   * @param params.countQuery - query builder for getting count
+   * @param params.searchQuery - query builder for search
+   * @param [params.dbQueryParameters] - db query parameters
+   */
+  protected buildTermQuery(params: {
+    countQuery: Knex.QueryBuilder,
+    searchQuery: Knex.QueryBuilder,
+    dbQueryParameters?: DbQueryParameters,
+  }) {
+    const table = typeToTable[this.type];
+    const { countQuery, searchQuery, dbQueryParameters } = params;
+    const { term = {} } = dbQueryParameters || this.dbQueryParameters;
+
+    Object.entries(term).forEach(([name, value]) => {
+      countQuery.where(`${table}.${name}`, value);
+      searchQuery.where(`${table}.${name}`, value);
+    });
+  }
+
   /**
    * Translate postgres records to api records
    *
@@ -96,7 +147,7 @@ class BaseSearch {
   }
 
   /**
-   * build and execute search query
+   * Build and execute search query
    *
    * @param testKnex - knex for testing
    * @returns search result