diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b444b5a4e5..08e527ac099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,102 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## Phase 2 Release + +### Breaking Changes + +- **CUMULUS-3698** + - GranuleSearch retrieving files/execution is toggled + by setting "includeFullRecord" field to 'true' in relevant api endpoint params + - GranuleSearch does *not* retrieve files/execution by default unless includeFullRecord is set to 'true' + - @cumulus/db function getExecutionArnByGranuleCumulusId is removed. To replace this function use getExecutionInfoByGranuleCumulusId with parameter executionColumns set to ['arn'] or unset (['arn'] is the default argument) + +### Migration Notes + +#### CUMULUS-3833 Migration of ReconciliationReports from DynamoDB to Postgres after Cumulus is upgraded. + +To invoke the Lambda and start the ReconciliationReport migration, you can use the AWS Console or CLI: + +```bash +aws lambda invoke --function-name $PREFIX-ReconciliationReportMigration $OUTFILE +``` + +- `PREFIX` is your Cumulus deployment prefix. +- `OUTFILE` (**optional**) is the filepath where the Lambda output will be saved. + + +#### CUMULUS-3967 + +External tooling making use of `searchContext` in the `GET` `/granules/` endpoint will need to update to make use of standard pagination via `limit` and `page` scrolling, as `searchContext` is no longer supported/is an ES specific feature. + +### Replace ElasticSearch Phase 2 + +- **CUMULUS-3967** + - Remove `searchContext` from API granules GET `/granules` endpoint. + - Update relevant tests to validate expected behavior utilizing postgres pagination +- **CUMULUS-3229** + - Remove ElasticSearch queries from Rule LIST endpoint +- **CUMULUS-3230** + - Remove ElasticSearch dependency from Rule Endpoints +- **CUMULUS-3231** + - Updated API `pdrs` `LIST` endpoint to query postgres +- **CUMULUS-3232** + - Update API PDR endpoints `DEL` and `GET` to not update Elasticsearch +- **CUMULUS-3233** + - Updated `providers` list api endpoint and added `ProviderSearch` class to query postgres + - Removed Elasticsearch dependency from `providers` endpoints +- **CUMULUS-3235** + - Updated `asyncOperations` api endpoint to query postgres +- **CUMULUS-3236** + - Update API AsyncOperation endpoints `POST` and `DEL` to not update + Elasticsearch + - Update `@cumlus/api/ecs/async-operation` to not update Elasticsearch index when + reporting status of async operation +- **CUMULUS-3698** + - GranuleSearch now can retrieve associated files for granules + - GranuleSearch now can retrieve latest associated execution for granules +- **CUMULUS-3806** + - Update `@cumulus/db/search` to allow for ordered collation as a + dbQueryParameter + - Update `@cumulus/db/search` to allow `dbQueryParameters.limit` to be set to + `null` to allow for optional unlimited page sizes in search results + - Update/add type annotations/logic fixes to `@cumulus/api` reconciliation report code + - Annotation/typing fixes to `@cumulus/cmr-client` + - Typing fixes to `@cumulus/db` + - Re-enable Reconciliation Report integration tests + - Update `@cumulus/client/CMR.getToken` to throw if a non-launchpad token is requested without a username + - Update `Inventory` and `Granule Not Found` reports to query postgreSQL + database instead of elasticsearch + - Update `@cumulus/db/lib/granule.getGranulesByApiPropertiesQuery` to + allow order by collation to be optionally specified + - Update `@cumulus/db/lib/granule.getGranulesByApiPropertiesQuery` to + be parameterized and include a modifier on `temporalBoundByCreatedAt` + - Remove endpoint call to and all tests for Internal Reconciliation Reports + and updated API to throw an error if report is requested + - Update Orca reconciliation reports to pull granules for comparison from + postgres via `getGranulesByApiPropertiesQuery` +- **CUMULUS-3837** + - Added `reconciliation_reports` table in RDS, including indexes + - Created pg model, types, and translation for `reconciliationReports` in `@cumulus/db` +- **CUMULUS-3833** + - Created api types for `reconciliation_reports` in `@cumulus/types/api` + - Updated reconciliation reports lambda to write to new RDS table instead of Dynamo + - Updated `@cumulus/api/endpoints/reconciliation-reports` `getReport` and `deleteReport` to work with the new RDS table instead of Dynamo +- **CUMULUS-3718** + - Updated `reconciliation_reports` list api endpoint and added `ReconciliationReportSearch` class to query postgres + - Added `reconciliationReports` type to stats endpoint, so `aggregate` query will work for reconciliation reports +- **CUMULUS-3859** + - Updated `@cumulus/api/bin/serveUtils` to no longer add records to ElasticSearch + - Removed ElasticSearch from local API server code + - Updated CollectionSearch to filter granule fields in addition to time frame for active collections +- **CUMULUS-3847** + - remove remaining ES indexing in code and tests + - for asyncOperations test data, change any ES related values to other options + - remove code from `@cumulus/api/lambdas/cleanExecutions` leaving a dummy handler, as the code worked with ES. lambda will be rewritten with CUMULUS-3982 + - remove `@cumulus/api/endpoints/elasticsearch`, `@cumulus/api/lambdas/bootstrap`, and `@cumulus/api/lambdas/index-from-database` +- **CUMULUS-3983** + - Removed elasticsearch references used in in cumulus `tf-modules` + ## [Unreleased] ### Breaking Changes diff --git a/audit-ci.json b/audit-ci.json index ada4b945034..939eba33e1e 100644 --- a/audit-ci.json +++ b/audit-ci.json @@ -3,12 +3,6 @@ "pass-enoaudit": true, "retry-count": 20, "allowlist": [ - { - "GHSA-776f-qx25-q3cc": { - "active": true, - "expiry": "1 July 2023 11:00" - } - }, "jsonpath-plus", "semver" ] diff --git a/example/cumulus-tf/main.tf b/example/cumulus-tf/main.tf index b799378ac90..ef68e6e5c2c 100644 --- a/example/cumulus-tf/main.tf +++ b/example/cumulus-tf/main.tf @@ -32,10 +32,6 @@ provider "aws" { locals { tags = merge(var.tags, { Deployment = var.prefix }) - elasticsearch_alarms = lookup(data.terraform_remote_state.data_persistence.outputs, "elasticsearch_alarms", []) - elasticsearch_domain_arn = lookup(data.terraform_remote_state.data_persistence.outputs, "elasticsearch_domain_arn", null) - elasticsearch_hostname = lookup(data.terraform_remote_state.data_persistence.outputs, "elasticsearch_hostname", null) - elasticsearch_security_group_id = lookup(data.terraform_remote_state.data_persistence.outputs, "elasticsearch_security_group_id", "") protected_bucket_names = [for k, v in var.buckets : v.name if v.type == "protected"] public_bucket_names = [for k, v in var.buckets : v.name if v.type == "public"] rds_security_group = lookup(data.terraform_remote_state.data_persistence.outputs, "rds_security_group", "") @@ -110,8 +106,6 @@ module "cumulus" { urs_client_id = var.urs_client_id urs_client_password = var.urs_client_password - es_request_concurrency = var.es_request_concurrency - metrics_es_host = var.metrics_es_host metrics_es_password = var.metrics_es_password metrics_es_username = var.metrics_es_username @@ -157,13 +151,6 @@ module "cumulus" { system_bucket = var.system_bucket buckets = var.buckets - elasticsearch_remove_index_alias_conflict = var.elasticsearch_remove_index_alias_conflict - elasticsearch_alarms = local.elasticsearch_alarms - elasticsearch_domain_arn = local.elasticsearch_domain_arn - elasticsearch_hostname = local.elasticsearch_hostname - elasticsearch_security_group_id = local.elasticsearch_security_group_id - es_index_shards = var.es_index_shards - dynamo_tables = merge(data.terraform_remote_state.data_persistence.outputs.dynamo_tables, var.optional_dynamo_tables) default_log_retention_days = var.default_log_retention_days cloudwatch_log_retention_periods = var.cloudwatch_log_retention_periods @@ -195,8 +182,6 @@ module "cumulus" { api_gateway_stage = var.api_gateway_stage archive_api_reserved_concurrency = var.api_reserved_concurrency - elasticsearch_client_config = var.elasticsearch_client_config - # Thin Egress App settings. Uncomment to use TEA. # must match stage_name variable for thin-egress-app module # tea_api_gateway_stage = local.tea_stage_name diff --git a/example/cumulus-tf/variables.tf b/example/cumulus-tf/variables.tf index 53db43ad32a..e428178c264 100644 --- a/example/cumulus-tf/variables.tf +++ b/example/cumulus-tf/variables.tf @@ -215,24 +215,6 @@ variable "ecs_include_docker_cleanup_cronjob" { default = false } -variable "elasticsearch_client_config" { - description = "Configuration parameters for Elasticsearch client for cumulus tasks" - type = map(string) - default = {} -} - -variable "elasticsearch_remove_index_alias_conflict" { - type = bool - default = true - description = "NOTE -- THIS SHOULD NEVER BE SET TO TRUE BY DEFAULT IN PRODUCTION SITUATIONS, we've set it to true here for dev only -- Set to false to not allow cumulus deployment bootstrap lambda to remove existing ES index named 'cumulus-alias'." -} - -variable "es_request_concurrency" { - type = number - default = 10 - description = "Maximum number of concurrent requests to send to Elasticsearch. Used in index-from-database operation" -} - variable "key_name" { type = string default = null @@ -318,12 +300,6 @@ variable "tags" { default = {} } -variable "es_index_shards" { - description = "The number of shards for the Elasticsearch index" - type = number - default = 2 -} - variable "pdr_node_name_provider_bucket" { type = string description = "The name of the common bucket used as an S3 provider for PDR NODE_NAME tests" @@ -350,7 +326,7 @@ variable "rds_admin_access_secret_arn" { variable "async_operation_image_version" { description = "docker image version to use for Cumulus async operations tasks" type = string - default = "52" + default = "53" } variable "cumulus_process_activity_version" { diff --git a/example/deployments/cumulus/sandbox.tfvars b/example/deployments/cumulus/sandbox.tfvars index 106ee0f8c1c..680caba85e5 100644 --- a/example/deployments/cumulus/sandbox.tfvars +++ b/example/deployments/cumulus/sandbox.tfvars @@ -45,11 +45,6 @@ csdap_host_url = "https://auth.csdap.uat.earthdatacloud.nasa.gov" default_s3_multipart_chunksize_mb = 128 -elasticsearch_client_config = { - create_reconciliation_report_es_scroll_duration = "8m" - create_reconciliation_report_es_scroll_size = 1500 -} - launchpad_api = "https://api.launchpad.nasa.gov/icam/api/sm/v1" launchpad_certificate = "launchpad.pfx" diff --git a/example/spec/helpers/granuleUtils.js b/example/spec/helpers/granuleUtils.js index 8e9e5af55ab..0cdd838b86c 100644 --- a/example/spec/helpers/granuleUtils.js +++ b/example/spec/helpers/granuleUtils.js @@ -238,8 +238,6 @@ const waitForGranuleRecordUpdatedInList = async (stackName, granule, additionalQ 'beginningDateTime', 'endingDateTime', 'error', - 'execution', // TODO remove after CUMULUS-3698 - 'files', // TODO -2714 this should be removed 'lastUpdateDateTime', 'productionDateTime', 'updatedAt', @@ -255,7 +253,8 @@ const waitForGranuleRecordUpdatedInList = async (stackName, granule, additionalQ }); const results = JSON.parse(resp.body).results; if (results && results.length === 1) { - // TODO - CUMULUS-2714 key sort both files objects for comparison + results[0].files.sort((a, b) => a.cumulus_id - b.cumulus_id); + granule.files.sort((a, b) => a.cumulus_id - b.cumulus_id); const granuleMatches = isEqual(omit(results[0], fieldsIgnored), omit(granule, fieldsIgnored)); if (!granuleMatches) { diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js index ea3a35db7c4..16301e7b2ff 100644 --- a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js +++ b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js @@ -111,13 +111,10 @@ const createActiveCollection = async (prefix, sourceBucket) => { const sourcePath = `${prefix}/tmp/${randomId('test-')}`; // Create the collection - const newCollection = await createCollection( - prefix, - { - duplicateHandling: 'error', - process: 'modis', - } - ); + const newCollection = await createCollection(prefix, { + duplicateHandling: 'error', + process: 'modis', + }); // Create the S3 provider const provider = await createProvider(prefix, { host: sourceBucket }); @@ -150,7 +147,12 @@ const createActiveCollection = async (prefix, sourceBucket) => { }; const workflowExecution = await buildAndExecuteWorkflow( - prefix, sourceBucket, 'IngestGranule', newCollection, provider, inputPayload + prefix, + sourceBucket, + 'IngestGranule', + newCollection, + provider, + inputPayload ); ingestGranuleExecutionArn = workflowExecution.executionArn; @@ -160,7 +162,10 @@ const createActiveCollection = async (prefix, sourceBucket) => { { prefix, granuleId: inputPayload.granules[0].granuleId, - collectionId: constructCollectionId(newCollection.name, newCollection.version), + collectionId: constructCollectionId( + newCollection.name, + newCollection.version + ), }, 'completed' ); @@ -172,10 +177,15 @@ const createActiveCollection = async (prefix, sourceBucket) => { status: 'completed', }); - await getGranuleWithStatus({ prefix, + await getGranuleWithStatus({ + prefix, granuleId, - collectionId: constructCollectionId(newCollection.name, newCollection.version), - status: 'completed' }); + collectionId: constructCollectionId( + newCollection.name, + newCollection.version + ), + status: 'completed', + }); return newCollection; }; @@ -272,8 +282,10 @@ async function updateGranuleFile(prefix, granule, regex, replacement) { const waitForCollectionRecordsInList = async (stackName, collectionIds, additionalQueryParams = {}) => await pWaitFor( async () => { // Verify the collection is returned when listing collections - const collsResp = await getCollections({ prefix: stackName, - query: { _id__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 } }); + const collsResp = await getCollections({ + prefix: stackName, + query: { _id__in: collectionIds.join(','), ...additionalQueryParams, limit: 30 }, + }); const results = get(JSON.parse(collsResp.body), 'results', []); const ids = results.map((c) => constructCollectionId(c.name, c.version)); return isEqual(ids.sort(), collectionIds.sort()); @@ -332,7 +344,6 @@ describe('When there are granule differences and granule reconciliation is run', config = await loadConfig(); - process.env.ReconciliationReportsTable = `${config.stackName}-ReconciliationReportsTable`; process.env.CMR_ENVIRONMENT = 'UAT'; cmrClient = await createCmrClient(config); @@ -348,12 +359,13 @@ describe('When there are granule differences and granule reconciliation is run', const testId = createTimestampedTestId(config.stackName, 'CreateReconciliationReport'); testSuffix = createTestSuffix(testId); testDataFolder = createTestDataPath(testId); - + const apiParams = { + includeFullRecord: 'true', + }; console.log('XXX Waiting for setupCollectionAndTestData'); await setupCollectionAndTestData(config, testSuffix, testDataFolder); - console.log('XXX Completed for setupCollectionAndTestData'); + console.log('XXX Completed setupCollectionAndTestData'); - // Write an extra file to the DynamoDB Files table extraFileInDb = { bucket: protectedBucket, key: randomString(), @@ -387,8 +399,8 @@ describe('When there are granule differences and granule reconciliation is run', collectionId, constructCollectionId(extraCumulusCollection.name, extraCumulusCollection.version), ]; - await waitForCollectionRecordsInList(config.stackName, collectionIds, { timestamp__from: ingestTime }); + console.log('XXXXX Completed collections in list'); // update one of the granule files in database so that that file won't match with CMR console.log('XXXXX Waiting for getGranule()'); @@ -398,7 +410,11 @@ describe('When there are granule differences and granule reconciliation is run', collectionId, }); console.log('XXXXX Completed for getGranule()'); - await waitForGranuleRecordUpdatedInList(config.stackName, granuleBeforeUpdate); + await waitForGranuleRecordUpdatedInList( + config.stackName, + granuleBeforeUpdate, + apiParams + ); console.log(`XXXXX Waiting for updateGranuleFile(${publishedGranuleId})`); ({ originalGranuleFile, updatedGranuleFile } = await updateGranuleFile( config.stackName, @@ -406,7 +422,7 @@ describe('When there are granule differences and granule reconciliation is run', /jpg$/, 'jpg2' )); - console.log(`XXXXX Completed for updateGranuleFile(${publishedGranuleId})`); + console.log(`XXXXX Completed updateGranuleFile(${publishedGranuleId})`); const [dbGranule, granuleAfterUpdate] = await Promise.all([ getGranule({ prefix: config.stackName, granuleId: dbGranuleId, collectionId }), @@ -414,9 +430,18 @@ describe('When there are granule differences and granule reconciliation is run', ]); console.log('XXXX Waiting for granules updated in list'); await Promise.all([ - waitForGranuleRecordUpdatedInList(config.stackName, dbGranule), - waitForGranuleRecordUpdatedInList(config.stackName, granuleAfterUpdate), + waitForGranuleRecordUpdatedInList( + config.stackName, + dbGranule, + apiParams + ), + waitForGranuleRecordUpdatedInList( + config.stackName, + granuleAfterUpdate, + apiParams + ), ]); + console.log('XXXX Completed granules updated in list'); } catch (error) { console.log(error); beforeAllFailed = error; @@ -427,8 +452,7 @@ describe('When there are granule differences and granule reconciliation is run', if (beforeAllFailed) fail(beforeAllFailed); }); - // TODO: fix tests in CUMULUS-3806 when CreateReconciliationReport lambda is changed to query postgres - xdescribe('Create an Inventory Reconciliation Report to monitor inventory discrepancies', () => { + describe('Create an Inventory Reconciliation Report to monitor inventory discrepancies', () => { // report record in db and report in s3 let reportRecord; let report; @@ -612,114 +636,6 @@ describe('When there are granule differences and granule reconciliation is run', }); }); - // TODO: the internal report functionality will be removed after collections/granules is changed to no longer use ES - xdescribe('Create an Internal Reconciliation Report to monitor internal discrepancies', () => { - // report record in db and report in s3 - let reportRecord; - let report; - let internalReportAsyncOperationId; - - afterAll(async () => { - if (internalReportAsyncOperationId) { - await deleteAsyncOperation({ prefix: config.stackName, asyncOperationId: internalReportAsyncOperationId }); - } - }); - - it('generates an async operation through the Cumulus API', async () => { - if (beforeAllFailed) fail(beforeAllFailed); - const request = { - reportType: 'Internal', - reportName: randomId('InternalReport'), - startTimestamp, - endTimestamp: moment.utc().format(), - collectionId, - granuleId: [publishedGranuleId, dbGranuleId, randomId('granuleId')], - provider: [randomId('provider'), `s3_provider${testSuffix}`], - }; - const response = await reconciliationReportsApi.createReconciliationReport({ - prefix: config.stackName, - request, - }); - - const responseBody = JSON.parse(response.body); - internalReportAsyncOperationId = responseBody.id; - console.log('internalReportAsyncOperationId', internalReportAsyncOperationId); - expect(response.statusCode).toBe(202); - }); - - it('generates reconciliation report through the Cumulus API', async () => { - if (beforeAllFailed) fail(beforeAllFailed); - let asyncOperation; - try { - asyncOperation = await waitForAsyncOperationStatus({ - id: internalReportAsyncOperationId, - status: 'SUCCEEDED', - stackName: config.stackName, - retryOptions: { - retries: 60, - factor: 1.08, - }, - }); - } catch (error) { - fail(error); - } - expect(asyncOperation.operationType).toBe('Reconciliation Report'); - reportRecord = JSON.parse(asyncOperation.output); - }); - - it('fetches a reconciliation report through the Cumulus API', async () => { - if (beforeAllFailed) fail(beforeAllFailed); - const reportContent = await fetchReconciliationReport(config.stackName, reportRecord.name); - report = JSON.parse(reportContent); - expect(report.reportType).toBe('Internal'); - expect(report.status).toBe('SUCCESS'); - }); - - it('generates a report showing number of collections that are in both ES and DB', () => { - if (beforeAllFailed) fail(beforeAllFailed); - expect(report.collections.okCount).toBe(1); - expect(report.collections.withConflicts.length).toBe(0); - expect(report.collections.onlyInEs.length).toBe(0); - expect(report.collections.onlyInDb.length).toBe(0); - }); - - it('generates a report showing number of granules that are in both ES and DB', () => { - if (beforeAllFailed) fail(beforeAllFailed); - expect(report.granules.okCount).toBe(2); - expect(report.granules.withConflicts.length).toBe(0); - if (report.granules.withConflicts.length !== 0) { - console.log(`XXXX ${JSON.stringify(report.granules.withConflicts)}`); - } - expect(report.granules.onlyInEs.length).toBe(0); - expect(report.granules.onlyInDb.length).toBe(0); - }); - - it('deletes a reconciliation report through the Cumulus API', async () => { - if (beforeAllFailed) fail(beforeAllFailed); - await reconciliationReportsApi.deleteReconciliationReport({ - prefix: config.stackName, - name: reportRecord.name, - }); - - const parsed = parseS3Uri(reportRecord.location); - const exists = await fileExists(parsed.Bucket, parsed.Key); - expect(exists).toBeFalse(); - - let responseError; - try { - await reconciliationReportsApi.getReconciliationReport({ - prefix: config.stackName, - name: reportRecord.name, - }); - } catch (error) { - responseError = error; - } - - expect(responseError.statusCode).toBe(404); - expect(JSON.parse(responseError.apiMessage).message).toBe(`No record found for ${reportRecord.name}`); - }); - }); - describe('Creates \'Granule Inventory\' reports.', () => { let reportRecord; let reportArray; @@ -834,8 +750,7 @@ describe('When there are granule differences and granule reconciliation is run', }); }); - // TODO: fix tests in CUMULUS-3806 when CreateReconciliationReport lambda is changed to query postgres - xdescribe('Create an ORCA Backup Reconciliation Report to monitor ORCA backup discrepancies', () => { + describe('Create an ORCA Backup Reconciliation Report to monitor ORCA backup discrepancies', () => { // report record in db and report in s3 let reportRecord; let report; @@ -916,6 +831,8 @@ describe('When there are granule differences and granule reconciliation is run', expect(granules.conflictFilesCount).toBe(6); expect(granules.onlyInCumulus.length).toBe(1); expect(granules.onlyInCumulus[0].granuleId).toBe(dbGranuleId); + expect(granules.onlyInCumulus[0].collectionId).toBe(collectionId); + expect(granules.onlyInCumulus[0].provider).toBe(`s3_provider${testSuffix}`); expect(granules.onlyInCumulus[0].okFilesCount).toBe(1); expect(granules.onlyInCumulus[0].cumulusFilesCount).toBe(5); expect(granules.onlyInCumulus[0].orcaFilesCount).toBe(0); @@ -927,6 +844,8 @@ describe('When there are granule differences and granule reconciliation is run', } expect(granules.withConflicts.length).toBe(1); expect(granules.withConflicts[0].granuleId).toBe(publishedGranuleId); + expect(granules.withConflicts[0].collectionId).toBe(collectionId); + expect(granules.withConflicts[0].provider).toBe(`s3_provider${testSuffix}`); expect(granules.withConflicts[0].okFilesCount).toBe(4); expect(granules.withConflicts[0].cumulusFilesCount).toBe(5); expect(granules.withConflicts[0].orcaFilesCount).toBe(4); diff --git a/example/spec/parallel/testAPI/granuleSpec.js b/example/spec/parallel/testAPI/granuleSpec.js index e9d170fa9e6..afce7a9412c 100644 --- a/example/spec/parallel/testAPI/granuleSpec.js +++ b/example/spec/parallel/testAPI/granuleSpec.js @@ -183,8 +183,26 @@ describe('The Granules API', () => { }); const searchedGranule = JSON.parse(searchResults.body).results[0]; - // TODO CUMULUS-3698 includes files - expect(searchedGranule).toEqual(jasmine.objectContaining(omit(randomGranuleRecord, 'files'))); + expect(searchedGranule).toEqual(jasmine.objectContaining({ + ...randomGranuleRecord, + files: [], + })); + }); + it('can search the granule including files via the API.', async () => { + if (beforeAllError) { + fail(beforeAllError); + } + + const searchResults = await waitForListGranulesResult({ + prefix, + query: { + granuleId: randomGranuleRecord.granuleId, + includeFullRecord: 'true', + }, + }); + + const searchedGranule = JSON.parse(searchResults.body).results[0]; + expect(searchedGranule).toEqual(jasmine.objectContaining(randomGranuleRecord)); }); it('can modify the granule via API.', async () => { diff --git a/example/spec/serial/AsyncOperationRunnerFailingLambdaSpec.js b/example/spec/serial/AsyncOperationRunnerFailingLambdaSpec.js index adcea147676..29f5e5bfd63 100644 --- a/example/spec/serial/AsyncOperationRunnerFailingLambdaSpec.js +++ b/example/spec/serial/AsyncOperationRunnerFailingLambdaSpec.js @@ -49,7 +49,7 @@ describe('The AsyncOperation task runner executing a failing lambda function', ( id: asyncOperationId, taskArn: randomString(), description: 'Some description', - operationType: 'ES Index', + operationType: 'Bulk Granules', status: 'RUNNING', }; diff --git a/example/spec/serial/AsyncOperationRunnerNonExistentLambdaSpec.js b/example/spec/serial/AsyncOperationRunnerNonExistentLambdaSpec.js index 7306851040c..10e98ff91fe 100644 --- a/example/spec/serial/AsyncOperationRunnerNonExistentLambdaSpec.js +++ b/example/spec/serial/AsyncOperationRunnerNonExistentLambdaSpec.js @@ -43,7 +43,7 @@ describe('The AsyncOperation task runner running a non-existent lambda function' id: asyncOperationId, taskArn: randomString(), description: 'Some description', - operationType: 'ES Index', + operationType: 'Bulk Granules', status: 'RUNNING', }; diff --git a/example/spec/serial/AsyncOperationRunnerNonExistentPayloadSpec.js b/example/spec/serial/AsyncOperationRunnerNonExistentPayloadSpec.js index b12dc77c5aa..c9676b7449a 100644 --- a/example/spec/serial/AsyncOperationRunnerNonExistentPayloadSpec.js +++ b/example/spec/serial/AsyncOperationRunnerNonExistentPayloadSpec.js @@ -43,7 +43,7 @@ describe('The AsyncOperation task runner with a non-existent payload', () => { id: asyncOperationId, taskArn: randomString(), description: 'Some description', - operationType: 'ES Index', + operationType: 'Bulk Granules', status: 'RUNNING', }; diff --git a/example/spec/serial/AsyncOperationRunnerNonJsonPayloadSpec.js b/example/spec/serial/AsyncOperationRunnerNonJsonPayloadSpec.js index 77403068625..c05dbfc9ab9 100644 --- a/example/spec/serial/AsyncOperationRunnerNonJsonPayloadSpec.js +++ b/example/spec/serial/AsyncOperationRunnerNonJsonPayloadSpec.js @@ -52,7 +52,7 @@ describe('The AsyncOperation task runner with a non-JSON payload', () => { id: asyncOperationId, taskArn: randomString(), description: 'Some description', - operationType: 'ES Index', + operationType: 'Kinesis Replay', status: 'RUNNING', }; diff --git a/example/spec/serial/AsyncOperationRunnerSuccessfulLambdaSpec.js b/example/spec/serial/AsyncOperationRunnerSuccessfulLambdaSpec.js index 554a72e4283..c497b91dbfc 100644 --- a/example/spec/serial/AsyncOperationRunnerSuccessfulLambdaSpec.js +++ b/example/spec/serial/AsyncOperationRunnerSuccessfulLambdaSpec.js @@ -4,7 +4,7 @@ const { waitUntilTasksStopped } = require('@aws-sdk/client-ecs'); const get = require('lodash/get'); const { v4: uuidv4 } = require('uuid'); -const { createAsyncOperation, deleteAsyncOperation, listAsyncOperations } = require('@cumulus/api-client/asyncOperations'); +const { createAsyncOperation, deleteAsyncOperation, getAsyncOperation } = require('@cumulus/api-client/asyncOperations'); const { startECSTask } = require('@cumulus/async-operations'); const { ecs, s3 } = require('@cumulus/aws-client/services'); const { randomString } = require('@cumulus/common/test-utils'); @@ -50,7 +50,7 @@ describe('The AsyncOperation task runner executing a successful lambda function' const asyncOperationObject = { description: 'Some description', - operationType: 'ES Index', + operationType: 'Bulk Granules', id: asyncOperationId, taskArn: randomString(), status: 'RUNNING', @@ -108,15 +108,10 @@ describe('The AsyncOperation task runner executing a successful lambda function' it('returns the updated record from GET /asyncOperations', async () => { if (beforeAllError) fail(beforeAllError); else { - const response = await listAsyncOperations({ + const record = await getAsyncOperation({ prefix: config.stackName, - query: { - id: asyncOperationId, - }, + asyncOperationId, }); - const { results } = JSON.parse(response.body); - expect(results.length).toEqual(1); - const [record] = results; expect(record.status).toEqual('SUCCEEDED'); const parsedOutput = JSON.parse(record.output); expect(parsedOutput).toEqual([1, 2, 3]); diff --git a/lambdas/migration-helper-async-operation/main.tf b/lambdas/migration-helper-async-operation/main.tf index c89559717b5..bdf16782287 100644 --- a/lambdas/migration-helper-async-operation/main.tf +++ b/lambdas/migration-helper-async-operation/main.tf @@ -19,7 +19,6 @@ resource "aws_lambda_function" "migration_helper_async_operation" { createTimeoutMillis = var.rds_connection_timing_configuration.createTimeoutMillis databaseCredentialSecretArn = var.rds_user_access_secret_arn EcsCluster = var.ecs_cluster_name - ES_HOST = var.elasticsearch_hostname idleTimeoutMillis = var.rds_connection_timing_configuration.idleTimeoutMillis DlaMigrationLambda = var.dla_migration_function_arn reapIntervalMillis = var.rds_connection_timing_configuration.reapIntervalMillis @@ -35,7 +34,6 @@ resource "aws_lambda_function" "migration_helper_async_operation" { security_group_ids = compact([ aws_security_group.migration_helper_async_operation[0].id, var.rds_security_group_id, - var.elasticsearch_security_group_id ]) } } diff --git a/lambdas/migration-helper-async-operation/variables.tf b/lambdas/migration-helper-async-operation/variables.tf index 1c033d09c27..9696f6cf1d9 100644 --- a/lambdas/migration-helper-async-operation/variables.tf +++ b/lambdas/migration-helper-async-operation/variables.tf @@ -14,14 +14,6 @@ variable "ecs_cluster_name" { type = string } -variable "elasticsearch_hostname" { - type = string -} - -variable "elasticsearch_security_group_id" { - description = "Security Group ID For Elasticsearch (OpenSearch)" -} - variable "ecs_execution_role_arn" { description = "ARN of IAM role for initializing ECS tasks" type = string diff --git a/lambdas/reconciliation-report-migration/.nycrc.json b/lambdas/reconciliation-report-migration/.nycrc.json new file mode 100644 index 00000000000..d7000c7c12b --- /dev/null +++ b/lambdas/reconciliation-report-migration/.nycrc.json @@ -0,0 +1,7 @@ +{ + "extends": "../../nyc.config.js", + "lines": 95.0, + "branches": 80.0, + "statements": 95.0, + "functions": 98.0 +} \ No newline at end of file diff --git a/lambdas/reconciliation-report-migration/README.md b/lambdas/reconciliation-report-migration/README.md new file mode 100644 index 00000000000..b117ad7a914 --- /dev/null +++ b/lambdas/reconciliation-report-migration/README.md @@ -0,0 +1,18 @@ +# ReconciliationReportMigration Lambda + +The lambda migrates existing ReconciliationReports data from DynamoDB to PostgreSQL. + +To invoke the Lambda and start the ReconciliationReport migration, you can use the AWS Console or CLI: + +```bash +aws lambda invoke --function-name $PREFIX-ReconciliationReportMigration $OUTFILE +``` + +- `PREFIX` is your Cumulus deployment prefix. +- `OUTFILE` (**optional**) is the filepath where the Lambda output will be saved. + +The result will be a migration summary. For example: + +``` +{"reconciliation_reports":{"total_dynamo_db_records":36,"migrated":36,"failed":0,"skipped":0}} +``` diff --git a/lambdas/reconciliation-report-migration/iam.tf b/lambdas/reconciliation-report-migration/iam.tf new file mode 100644 index 00000000000..3fae7f2bd89 --- /dev/null +++ b/lambdas/reconciliation-report-migration/iam.tf @@ -0,0 +1,70 @@ +data "aws_iam_policy_document" "lambda_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "reconciliation_report_migration" { + name = "${var.prefix}-reconciliation-report-migration" + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json + permissions_boundary = var.permissions_boundary_arn + + tags = var.tags +} + +data "aws_iam_policy_document" "reconciliation_report_migration" { + statement { + actions = [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ] + resources = ["*"] + } + + statement { + actions = [ + "dynamodb:Scan", + ] + resources = [ + var.dynamo_tables.reconciliation_reports.arn, + ] + } + + statement { + actions = [ + "secretsmanager:GetSecretValue" + ] + resources = [var.rds_user_access_secret_arn] + } +} + +resource "aws_iam_role_policy" "reconciliation_report_migration" { + name = "${var.prefix}_reconciliation_report_migration" + role = aws_iam_role.reconciliation_report_migration.id + policy = data.aws_iam_policy_document.reconciliation_report_migration.json +} + +resource "aws_security_group" "reconciliation_report_migration" { + count = length(var.lambda_subnet_ids) == 0 ? 0 : 1 + + name = "${var.prefix}-reconciliation-report-migration" + vpc_id = var.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = var.tags +} diff --git a/lambdas/reconciliation-report-migration/main.tf b/lambdas/reconciliation-report-migration/main.tf new file mode 100644 index 00000000000..bb7e543fe16 --- /dev/null +++ b/lambdas/reconciliation-report-migration/main.tf @@ -0,0 +1,35 @@ +locals { + lambda_path = "${path.module}/dist/webpack/lambda.zip" +} + +resource "aws_lambda_function" "reconciliation_report_migration" { + function_name = "${var.prefix}-ReconciliationReportMigration" + filename = local.lambda_path + source_code_hash = filebase64sha256(local.lambda_path) + handler = "index.handler" + role = aws_iam_role.reconciliation_report_migration.arn + runtime = "nodejs20.x" + timeout = lookup(var.lambda_timeouts, "ReconciliationReportMigration", 900) + memory_size = lookup(var.lambda_memory_sizes, "ReconciliationReportMigration", 1024) + + environment { + variables = { + databaseCredentialSecretArn = var.rds_user_access_secret_arn + ReconciliationReportsTable = var.dynamo_tables.reconciliation_reports.name + stackName = var.prefix + } + } + + dynamic "vpc_config" { + for_each = length(var.lambda_subnet_ids) == 0 ? [] : [1] + content { + subnet_ids = var.lambda_subnet_ids + security_group_ids = compact([ + aws_security_group.reconciliation_report_migration[0].id, + var.rds_security_group_id + ]) + } + } + + tags = var.tags +} diff --git a/lambdas/reconciliation-report-migration/outputs.tf b/lambdas/reconciliation-report-migration/outputs.tf new file mode 100644 index 00000000000..122c24f1abc --- /dev/null +++ b/lambdas/reconciliation-report-migration/outputs.tf @@ -0,0 +1,3 @@ +output "reconciliation_report_migration_function_arn" { + value = aws_lambda_function.reconciliation_report_migration.arn +} diff --git a/lambdas/reconciliation-report-migration/package.json b/lambdas/reconciliation-report-migration/package.json new file mode 100644 index 00000000000..98bcf797212 --- /dev/null +++ b/lambdas/reconciliation-report-migration/package.json @@ -0,0 +1,45 @@ +{ + "name": "@cumulus/reconciliation-report-migration", + "version": "19.1.0", + "description": "Lambda function for reconciliation report migration from DynamoDB to Postgres", + "author": "Cumulus Authors", + "license": "Apache-2.0", + "engines": { + "node": ">=20.12.2" + }, + "private": true, + "main": "./dist/lambda/index.js", + "types": "./dist/lambda/index.d.ts", + "scripts": { + "clean": "rm -rf dist", + "build": "rm -rf dist && mkdir dist && npm run prepare && npm run webpack", + "build-lambda-zip": "cd dist/webpack && node ../../../../bin/zip.js lambda.zip index.js", + "package": "npm run clean && npm run prepare && npm run webpack && npm run build-lambda-zip", + "test": "../../node_modules/.bin/ava", + "test:ci": "../../scripts/run_package_ci_unit.sh", + "test:coverage": "../../node_modules/.bin/nyc npm test", + "prepare": "npm run tsc", + "tsc": "../../node_modules/.bin/tsc", + "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles", + "webpack": "../../node_modules/.bin/webpack" + }, + "ava": { + "files": [ + "tests/**/*.js" + ], + "timeout": "15m", + "failFast": true + }, + "dependencies": { + "@cumulus/api": "19.1.0", + "@cumulus/aws-client": "19.1.0", + "@cumulus/common": "19.1.0", + "@cumulus/db": "19.1.0", + "@cumulus/errors": "19.1.0", + "@cumulus/logger": "19.1.0", + "@cumulus/types": "19.1.0", + "knex": "2.4.1", + "lodash": "^4.17.21", + "pg": "~8.12" + } +} diff --git a/lambdas/reconciliation-report-migration/src/index.ts b/lambdas/reconciliation-report-migration/src/index.ts new file mode 100644 index 00000000000..a5c394d28fa --- /dev/null +++ b/lambdas/reconciliation-report-migration/src/index.ts @@ -0,0 +1,24 @@ +import { getKnexClient } from '@cumulus/db'; +import Logger from '@cumulus/logger'; + +import { migrateReconciliationReports } from './reconciliation-reports'; +import { MigrationSummary } from './types'; + +const logger = new Logger({ sender: '@cumulus/reconciliation-report-migration' }); + +export interface HandlerEvent { + env?: NodeJS.ProcessEnv +} + +export const handler = async (event: HandlerEvent): Promise => { + const env = event.env ?? process.env; + const knex = await getKnexClient({ env }); + + try { + const migrationSummary = await migrateReconciliationReports(env, knex); + logger.info(JSON.stringify(migrationSummary)); + return { reconciliation_reports: migrationSummary }; + } finally { + await knex.destroy(); + } +}; diff --git a/lambdas/reconciliation-report-migration/src/reconciliation-reports.ts b/lambdas/reconciliation-report-migration/src/reconciliation-reports.ts new file mode 100644 index 00000000000..32dec7c570e --- /dev/null +++ b/lambdas/reconciliation-report-migration/src/reconciliation-reports.ts @@ -0,0 +1,88 @@ +import { Knex } from 'knex'; + +import { DynamoDbSearchQueue } from '@cumulus/aws-client'; +import { envUtils } from '@cumulus/common'; +import { + ReconciliationReportPgModel, + translateApiReconReportToPostgresReconReport, +} from '@cumulus/db'; +import { RecordAlreadyMigrated, RecordDoesNotExist } from '@cumulus/errors'; +import Logger from '@cumulus/logger'; +import { ApiReconciliationReportRecord } from '@cumulus/types/api/reconciliation_reports'; + +import { MigrationResult } from './types'; + +const logger = new Logger({ sender: '@cumulus/data-migration/reconciliation-reports' }); + +export const migrateReconciliationReportRecord = async ( + dynamoRecord: ApiReconciliationReportRecord, + knex: Knex +): Promise => { + const reconReportPgModel = new ReconciliationReportPgModel(); + + let existingRecord; + try { + existingRecord = await reconReportPgModel.get(knex, { name: dynamoRecord.name }); + } catch (error) { + if (!(error instanceof RecordDoesNotExist)) { + throw error; + } + } + + if (existingRecord + && dynamoRecord.updatedAt + && existingRecord.updated_at >= new Date(dynamoRecord.updatedAt)) { + throw new RecordAlreadyMigrated(`Reconciliation report ${dynamoRecord.name} was already migrated, skipping`); + } + + const updatedRecord = translateApiReconReportToPostgresReconReport( + dynamoRecord + ); + + await reconReportPgModel.upsert(knex, updatedRecord); +}; + +export const migrateReconciliationReports = async ( + env: NodeJS.ProcessEnv, + knex: Knex +): Promise => { + const reconciliationReportsTable = envUtils.getRequiredEnvVar('ReconciliationReportsTable', env); + + const searchQueue = new DynamoDbSearchQueue({ + TableName: reconciliationReportsTable, + }); + + const migrationSummary = { + total_dynamo_db_records: 0, + migrated: 0, + failed: 0, + skipped: 0, + }; + + let record = await searchQueue.peek(); + /* eslint-disable no-await-in-loop */ + while (record) { + migrationSummary.total_dynamo_db_records += 1; + + try { + await migrateReconciliationReportRecord(record as any, knex); + migrationSummary.migrated += 1; + } catch (error) { + if (error instanceof RecordAlreadyMigrated) { + migrationSummary.skipped += 1; + } else { + migrationSummary.failed += 1; + logger.error( + `Could not create reconciliationReport record in RDS for Dynamo reconciliationReport name ${record.name}:`, + error + ); + } + } + + await searchQueue.shift(); + record = await searchQueue.peek(); + } + /* eslint-enable no-await-in-loop */ + logger.info(`successfully migrated ${migrationSummary.migrated} reconciliationReport records`); + return migrationSummary; +}; diff --git a/lambdas/reconciliation-report-migration/src/types.ts b/lambdas/reconciliation-report-migration/src/types.ts new file mode 100644 index 00000000000..08119110cae --- /dev/null +++ b/lambdas/reconciliation-report-migration/src/types.ts @@ -0,0 +1,10 @@ +export type MigrationResult = { + total_dynamo_db_records: number, + skipped: number, + migrated: number, + failed: number, +}; + +export type MigrationSummary = { + reconciliation_reports: MigrationResult +}; diff --git a/lambdas/reconciliation-report-migration/tests/test-index.js b/lambdas/reconciliation-report-migration/tests/test-index.js new file mode 100644 index 00000000000..b355c0b100d --- /dev/null +++ b/lambdas/reconciliation-report-migration/tests/test-index.js @@ -0,0 +1,104 @@ +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); + +const ReconciliationReport = require('@cumulus/api/models/reconciliation-reports'); + +const { + createBucket, + putJsonS3Object, + recursivelyDeleteS3Bucket, +} = require('@cumulus/aws-client/S3'); + +const { + generateLocalTestDb, + destroyLocalTestDb, + localStackConnectionEnv, + migrationDir, +} = require('@cumulus/db'); + +const { handler } = require('../dist/lambda'); +const testDbName = `reconciliation_report_migration_1_${cryptoRandomString({ length: 10 })}`; +const workflow = cryptoRandomString({ length: 10 }); + +test.before(async (t) => { + process.env = { + ...process.env, + ...localStackConnectionEnv, + PG_DATABASE: testDbName, + stackName: cryptoRandomString({ length: 10 }), + system_bucket: cryptoRandomString({ length: 10 }), + ReconciliationReportsTable: cryptoRandomString({ length: 10 }), + }; + + await createBucket(process.env.system_bucket); + + const workflowfile = `${process.env.stackName}/workflows/${workflow}.json`; + const messageTemplateKey = `${process.env.stackName}/workflow_template.json`; + + t.context.reconciliationReportsModel = new ReconciliationReport({ + stackName: process.env.stackName, + systemBucket: process.env.system_bucket, + }); + + await Promise.all([ + t.context.reconciliationReportsModel.createTable(), + ]); + + await Promise.all([ + putJsonS3Object( + process.env.system_bucket, + messageTemplateKey, + { meta: 'meta' } + ), + putJsonS3Object( + process.env.system_bucket, + workflowfile, + { testworkflow: 'workflow-config' } + ), + ]); + const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); + t.context.knex = knex; + t.context.knexAdmin = knexAdmin; +}); + +test.after.always(async (t) => { + await t.context.reconciliationReportsModel.deleteTable(); + + await recursivelyDeleteS3Bucket(process.env.system_bucket); + + await destroyLocalTestDb({ + knex: t.context.knex, + knexAdmin: t.context.knexAdmin, + testDbName, + }); +}); + +test('handler migrates reconciliation reports', async (t) => { + const { reconciliationReportsModel } = t.context; + + const fakeReconciliationReport = { + name: cryptoRandomString({ length: 5 }), + type: 'Granule Inventory', + status: 'Generated', + error: {}, + createdAt: (Date.now() - 1000), + updatedAt: Date.now(), + }; + + await Promise.all([ + reconciliationReportsModel.create(fakeReconciliationReport), + ]); + + t.teardown(() => reconciliationReportsModel.delete({ name: fakeReconciliationReport.name })); + + const call = await handler({}); + const expected = { + reconciliation_reports: { + failed: 0, + migrated: 1, + skipped: 0, + total_dynamo_db_records: 1, + }, + }; + t.deepEqual(call, expected); +}); diff --git a/lambdas/reconciliation-report-migration/tests/test-reconciliation-reports.js b/lambdas/reconciliation-report-migration/tests/test-reconciliation-reports.js new file mode 100644 index 00000000000..79afe2d4a58 --- /dev/null +++ b/lambdas/reconciliation-report-migration/tests/test-reconciliation-reports.js @@ -0,0 +1,245 @@ +const cryptoRandomString = require('crypto-random-string'); +const omit = require('lodash/omit'); +const test = require('ava'); + +const ReconciliationReport = require('@cumulus/api/models/reconciliation-reports'); +const { dynamodbDocClient } = require('@cumulus/aws-client/services'); +const { + createBucket, + recursivelyDeleteS3Bucket, +} = require('@cumulus/aws-client/S3'); +const { + generateLocalTestDb, + destroyLocalTestDb, + ReconciliationReportPgModel, + migrationDir, +} = require('@cumulus/db'); +const { RecordAlreadyMigrated } = require('@cumulus/errors'); + +const { + migrateReconciliationReportRecord, + migrateReconciliationReports, +} = require('../dist/lambda/reconciliation-reports'); + +const testDbName = `reconciliation_reports_migration_${cryptoRandomString({ length: 10 })}`; + +const generateFakeReconciliationReport = (params) => ({ + name: cryptoRandomString({ length: 5 }), + type: 'Granule Inventory', + status: 'Generated', + error: {}, + location: `s3://${cryptoRandomString({ length: 10 })}/${cryptoRandomString({ length: 10 })}`, + createdAt: (Date.now() - 1000), + updatedAt: Date.now(), + ...params, +}); + +let reconciliationReportsModel; + +test.before(async (t) => { + process.env.stackName = cryptoRandomString({ length: 10 }); + process.env.system_bucket = cryptoRandomString({ length: 10 }); + process.env.ReconciliationReportsTable = cryptoRandomString({ length: 10 }); + + await createBucket(process.env.system_bucket); + + reconciliationReportsModel = new ReconciliationReport({ + stackName: process.env.stackName, + systemBucket: process.env.system_bucket, + }); + await reconciliationReportsModel.createTable(); + + t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); + + const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); + t.context.knex = knex; + t.context.knexAdmin = knexAdmin; +}); + +test.afterEach.always(async (t) => { + await t.context.knex('reconciliation_reports').del(); +}); + +test.after.always(async (t) => { + await reconciliationReportsModel.deleteTable(); + await recursivelyDeleteS3Bucket(process.env.system_bucket); + await destroyLocalTestDb({ + knex: t.context.knex, + knexAdmin: t.context.knexAdmin, + testDbName, + }); +}); + +test.serial('migrateReconciliationReportRecord correctly migrates reconciliationReport record', async (t) => { + const { knex, reconciliationReportPgModel } = t.context; + + const fakeReconReport = generateFakeReconciliationReport(); + await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); + + const createdRecord = await reconciliationReportPgModel.get( + knex, + { name: fakeReconReport.name } + ); + + t.deepEqual( + omit(createdRecord, ['cumulus_id']), + omit({ + ...fakeReconReport, + created_at: new Date(fakeReconReport.createdAt), + updated_at: new Date(fakeReconReport.updatedAt), + }, ['createdAt', 'updatedAt']) + ); +}); + +test.serial('migrateReconciliationReportRecord correctly migrates reconciliationReport record where record.error is an object', async (t) => { + const error = { exception: 'there is an error' }; + const fakeReconReport = generateFakeReconciliationReport({ error }); + await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); + + const createdRecord = await t.context.knex.queryBuilder() + .select() + .table('reconciliation_reports') + .where({ name: fakeReconReport.name }) + .first(); + + t.deepEqual( + omit(createdRecord, ['cumulus_id']), + omit({ + ...fakeReconReport, + created_at: new Date(fakeReconReport.createdAt), + updated_at: new Date(fakeReconReport.updatedAt), + }, ['createdAt', 'updatedAt']) + ); +}); + +test.serial('migrateReconciliationReportRecord migrates reconciliationReport record with undefined nullables', async (t) => { + const { knex, reconciliationReportPgModel } = t.context; + + const fakeReconReport = generateFakeReconciliationReport(); + delete fakeReconReport.error; + delete fakeReconReport.location; + await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); + + const createdRecord = await reconciliationReportPgModel.get( + knex, + { name: fakeReconReport.name } + ); + + t.deepEqual( + omit(createdRecord, ['cumulus_id']), + omit({ + ...fakeReconReport, + error: null, + location: null, + created_at: new Date(fakeReconReport.createdAt), + updated_at: new Date(fakeReconReport.updatedAt), + }, ['createdAt', 'updatedAt']) + ); +}); + +test.serial('migrateReconciliationReportRecord throws RecordAlreadyMigrated error if already migrated record is newer', async (t) => { + const fakeReconReport = generateFakeReconciliationReport({ + updatedAt: Date.now(), + }); + + await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); + + const olderFakeReconReport = { + ...fakeReconReport, + updatedAt: Date.now() - 1000, // older than fakeReconReport + }; + + await t.throwsAsync( + migrateReconciliationReportRecord(olderFakeReconReport, t.context.knex), + { instanceOf: RecordAlreadyMigrated } + ); +}); + +test.serial('migrateReconciliationReportRecord updates an already migrated record if the updated date is newer', async (t) => { + const { knex, reconciliationReportPgModel } = t.context; + + const fakeReconReport = generateFakeReconciliationReport({ + updatedAt: Date.now() - 1000, + }); + await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); + + const newerFakeReconReport = generateFakeReconciliationReport({ + ...fakeReconReport, + updatedAt: Date.now(), + }); + await migrateReconciliationReportRecord(newerFakeReconReport, t.context.knex); + + const createdRecord = await reconciliationReportPgModel.get( + knex, + { name: fakeReconReport.name } + ); + + t.deepEqual(createdRecord.updated_at, new Date(newerFakeReconReport.updatedAt)); +}); + +test.serial('migrateReconciliationReports processes multiple reconciliation reports', async (t) => { + const { knex, reconciliationReportPgModel } = t.context; + + const fakeReconReport1 = generateFakeReconciliationReport(); + const fakeReconReport2 = generateFakeReconciliationReport(); + + await Promise.all([ + reconciliationReportsModel.create(fakeReconReport1), + reconciliationReportsModel.create(fakeReconReport2), + ]); + t.teardown(() => Promise.all([ + reconciliationReportsModel.delete({ name: fakeReconReport1.name }), + reconciliationReportsModel.delete({ name: fakeReconReport2.name }), + ])); + + const migrationSummary = await migrateReconciliationReports(process.env, t.context.knex); + t.deepEqual(migrationSummary, { + total_dynamo_db_records: 2, + skipped: 0, + failed: 0, + migrated: 2, + }); + + const records = await reconciliationReportPgModel.search( + knex, + {} + ); + t.is(records.length, 2); +}); + +test.serial('migrateReconciliationReports processes all non-failing records', async (t) => { + const { knex, reconciliationReportPgModel } = t.context; + + const fakeReconReport1 = generateFakeReconciliationReport(); + const fakeReconReport2 = generateFakeReconciliationReport(); + + // remove required source field so that record will fail + delete fakeReconReport1.status; + + await Promise.all([ + // Have to use Dynamo client directly because creating + // via model won't allow creation of an invalid record + dynamodbDocClient().put({ + TableName: process.env.ReconciliationReportsTable, + Item: fakeReconReport1, + }), + reconciliationReportsModel.create(fakeReconReport2), + ]); + t.teardown(() => Promise.all([ + reconciliationReportsModel.delete({ name: fakeReconReport1.name }), + reconciliationReportsModel.delete({ name: fakeReconReport2.name }), + ])); + + const migrationSummary = await migrateReconciliationReports(process.env, t.context.knex); + t.deepEqual(migrationSummary, { + total_dynamo_db_records: 2, + skipped: 0, + failed: 1, + migrated: 1, + }); + const records = await reconciliationReportPgModel.search( + knex, + {} + ); + t.is(records.length, 1); +}); diff --git a/lambdas/reconciliation-report-migration/tsconfig.json b/lambdas/reconciliation-report-migration/tsconfig.json new file mode 100644 index 00000000000..4b4ae9578ce --- /dev/null +++ b/lambdas/reconciliation-report-migration/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist/lambda", + "declaration": false, + "declarationMap": false, + "sourceMap": true, + "removeComments": true + }, + "include": ["src"], +} diff --git a/lambdas/reconciliation-report-migration/variables.tf b/lambdas/reconciliation-report-migration/variables.tf new file mode 100644 index 00000000000..ad4ff7463cf --- /dev/null +++ b/lambdas/reconciliation-report-migration/variables.tf @@ -0,0 +1,59 @@ +# Required + +variable "dynamo_tables" { + description = "A map of objects with the `arn` and `name` of every DynamoDB table for your Cumulus deployment." + type = map(object({ name = string, arn = string })) +} + +variable "permissions_boundary_arn" { + type = string +} + +variable "prefix" { + type = string +} + +variable "rds_user_access_secret_arn" { + description = "RDS User Database Login Credential Secret ID" + type = string +} + +variable "system_bucket" { + description = "The name of the S3 bucket to be used for staging deployment files" + type = string +} + +# Optional + +variable "lambda_memory_sizes" { + description = "Configurable map of memory sizes for lambdas" + type = map(number) + default = {} +} + +variable "lambda_timeouts" { + description = "Configurable map of timeouts for lambdas" + type = map(number) + default = {} +} + +variable "lambda_subnet_ids" { + type = list(string) + default = [] +} + +variable "rds_security_group_id" { + description = "RDS Security Group used for access to RDS cluster" + type = string + default = "" +} + +variable "tags" { + type = map(string) + default = {} +} + +variable "vpc_id" { + type = string + default = null +} diff --git a/lambdas/reconciliation-report-migration/versions.tf b/lambdas/reconciliation-report-migration/versions.tf new file mode 100644 index 00000000000..c62a4968cfd --- /dev/null +++ b/lambdas/reconciliation-report-migration/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + required_version = ">= 1.5" +} diff --git a/lambdas/reconciliation-report-migration/webpack.config.js b/lambdas/reconciliation-report-migration/webpack.config.js new file mode 100644 index 00000000000..b0a194e3834 --- /dev/null +++ b/lambdas/reconciliation-report-migration/webpack.config.js @@ -0,0 +1,53 @@ + +const path = require('path'); +const { IgnorePlugin } = require('webpack'); + +const ignoredPackages = [ + 'better-sqlite3', + 'mssql', + 'mssql/lib/base', + 'mssql/package.json', + 'mysql', + 'mysql2', + 'oracledb', + 'pg-native', + 'pg-query-stream', + 'sqlite3', + 'tedious' +]; + +module.exports = { + plugins: [ + new IgnorePlugin({ + resourceRegExp: new RegExp(`^(${ignoredPackages.join('|')})$`) + }), + ], + mode: 'production', + entry: './dist/lambda/index.js', + output: { + chunkFormat: false, + libraryTarget: 'commonjs2', + filename: 'index.js', + path: path.resolve(__dirname, 'dist', 'webpack') + }, + module: { + rules: [ + { + test: /\.js$/, + exclude: /node_modules/, + use: [ + { + loader: 'babel-loader', + options: { + cacheDirectory: true + }, + }, + ], + }, + ], + }, + target: 'node', + externals: [ + /@aws-sdk\// + ] +}; diff --git a/packages/api/app/routes.js b/packages/api/app/routes.js index cf655198c86..25a8f76de78 100644 --- a/packages/api/app/routes.js +++ b/packages/api/app/routes.js @@ -23,7 +23,6 @@ const stats = require('../endpoints/stats'); const version = require('../endpoints/version'); const workflows = require('../endpoints/workflows'); const dashboard = require('../endpoints/dashboard'); -const elasticsearch = require('../endpoints/elasticsearch'); const deadLetterArchive = require('../endpoints/dead-letter-archive'); const { launchpadProtectedAuth } = require('./launchpadAuth'); const launchpadSaml = require('../endpoints/launchpadSaml'); @@ -110,8 +109,6 @@ router.delete('/tokenDelete/:token', token.deleteTokenEndpoint); router.use('/dashboard', dashboard); -router.use('/elasticsearch', ensureAuthorized, elasticsearch.router); - // Catch and send the error message down (instead of just 500: internal server error) router.use(defaultErrorHandler); diff --git a/packages/api/bin/cli.js b/packages/api/bin/cli.js index cac11b2a74c..b5eea458b8a 100755 --- a/packages/api/bin/cli.js +++ b/packages/api/bin/cli.js @@ -54,7 +54,7 @@ program program .command('serve') .option('--stackName ', 'stackname to serve (defaults to "localrun")', undefined) - .option('--no-reseed', 'do not reseed dynamoDB and Elasticsearch with new data on start.') + .option('--no-reseed', 'do not reseed data stores with new data on start.') .description('Serves the local version of the Cumulus API') .action((cmd) => { serveApi(process.env.USERNAME, cmd.stackName, cmd.reseed).catch(console.error); diff --git a/packages/api/bin/serve.js b/packages/api/bin/serve.js index 591f04ae227..78d609fe7b6 100644 --- a/packages/api/bin/serve.js +++ b/packages/api/bin/serve.js @@ -19,18 +19,14 @@ const { const { constructCollectionId } = require('@cumulus/message/Collections'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); - const { ReconciliationReport } = require('../models'); const testUtils = require('../lib/testUtils'); const serveUtils = require('./serveUtils'); const { - setLocalEsVariables, localStackName, localSystemBucket, localUserName, - getESClientAndIndex, } = require('./local-test-defaults'); const workflowList = testUtils.getWorkflowList(); @@ -58,12 +54,6 @@ async function populateBucket(bucket, stackName) { } async function prepareServices(stackName, bucket) { - setLocalEsVariables(stackName); - console.log(process.env.ES_HOST); - await bootstrapElasticSearch({ - host: process.env.ES_HOST, - index: process.env.ES_INDEX, - }); await s3().createBucket({ Bucket: bucket }); const { TopicArn } = await createSnsTopic(randomId('topicName')); @@ -96,35 +86,7 @@ function checkEnvVariablesAreSet(moreRequiredEnvVars) { } /** - * erases Elasticsearch index - * @param {any} esClient - Elasticsearch client - * @param {any} esIndex - index to delete - */ -async function eraseElasticsearchIndices(esClient, esIndex) { - try { - await esClient.client.indices.delete({ index: esIndex }); - } catch (error) { - if (error.message !== 'index_not_found_exception') throw error; - } -} - -/** - * resets Elasticsearch and returns the client and index. - * - * @param {string} stackName - The name of local stack. Used to prefix stack resources. - * @returns {Object} - Elasticsearch client and index - */ -async function initializeLocalElasticsearch(stackName) { - const es = await getESClientAndIndex(stackName); - await eraseElasticsearchIndices(es.client, es.index); - return bootstrapElasticSearch({ - host: process.env.ES_HOST, - index: es.index, - }); -} - -/** - * Fill Postgres and Elasticsearch with fake records for testing. + * Fill Postgres with fake records for testing. * @param {string} stackName - The name of local stack. Used to prefix stack resources. * @param {string} user - username * @param {Object} knexOverride - Used to override knex object for testing @@ -140,7 +102,6 @@ async function createDBRecords(stackName, user, knexOverride) { const providerPgModel = new ProviderPgModel(); const rulePgModel = new RulePgModel(); - await initializeLocalElasticsearch(stackName); await serveUtils.resetPostgresDb(); if (user) { @@ -219,7 +180,7 @@ async function createDBRecords(stackName, user, knexOverride) { * @param {string} user - A username to add as an authorized user for the API. * @param {string} stackName - The name of local stack. Used to prefix stack resources. * @param {bool} reseed - boolean to control whether to load new data into - * Postgres and Elasticsearch. + * Postgres. */ async function serveApi(user, stackName = localStackName, reseed = true) { const port = process.env.PORT || 5001; @@ -320,18 +281,6 @@ async function serveDistributionApi(stackName = localStackName, done) { return distributionApp.listen(port, done); } -/** - * Resets Elasticsearch - * - * @param {string} stackName - defaults to local stack, 'localrun' - * @param {string} systemBucket - defaults to 'localbucket' - */ -function eraseDataStack( - stackName = localStackName -) { - return initializeLocalElasticsearch(stackName); -} - /** * Removes all additional data from tables and repopulates with original data. * @@ -353,7 +302,6 @@ async function resetTables( } module.exports = { - eraseDataStack, serveApi, serveDistributionApi, resetTables, diff --git a/packages/api/bin/serveUtils.js b/packages/api/bin/serveUtils.js index 76c96a4e73a..f3d6f277987 100644 --- a/packages/api/bin/serveUtils.js +++ b/packages/api/bin/serveUtils.js @@ -1,7 +1,6 @@ 'use strict'; const pEachSeries = require('p-each-series'); -const indexer = require('@cumulus/es-client/indexer'); const { AsyncOperationPgModel, CollectionPgModel, @@ -16,21 +15,21 @@ const { migrationDir, PdrPgModel, ProviderPgModel, + ReconciliationReportPgModel, RulePgModel, + translateApiAsyncOperationToPostgresAsyncOperation, translateApiCollectionToPostgresCollection, translateApiExecutionToPostgresExecution, translateApiGranuleToPostgresGranule, translateApiPdrToPostgresPdr, translateApiProviderToPostgresProvider, + translateApiReconReportToPostgresReconReport, translateApiRuleToPostgresRule, - translatePostgresExecutionToApiExecution, upsertGranuleWithExecutionJoinRecord, } = require('@cumulus/db'); const { log } = require('console'); -const models = require('../models'); const { createRuleTrigger } = require('../lib/rulesHelpers'); const { fakeGranuleFactoryV2 } = require('../lib/testUtils'); -const { getESClientAndIndex } = require('./local-test-defaults'); /** * Remove all records from api-related postgres tables @@ -46,6 +45,7 @@ async function erasePostgresTables(knex) { const granulesExecutionsPgModel = new GranulesExecutionsPgModel(); const pdrPgModel = new PdrPgModel(); const providerPgModel = new ProviderPgModel(); + const reconReportPgModel = new ReconciliationReportPgModel(); const rulePgModel = new RulePgModel(); await granulesExecutionsPgModel.delete(knex, {}); @@ -58,6 +58,7 @@ async function erasePostgresTables(knex) { await rulePgModel.delete(knex, {}); await collectionPgModel.delete(knex, {}); await providerPgModel.delete(knex, {}); + await reconReportPgModel.delete(knex, {}); } async function resetPostgresDb() { @@ -81,6 +82,22 @@ async function resetPostgresDb() { await erasePostgresTables(knex); } +async function addAsyncOperations(asyncOperations) { + const knex = await getKnexClient({ + env: { + ...envParams, + ...localStackConnectionEnv, + }, + }); + const asyncOperationPgModel = new AsyncOperationPgModel(); + return await Promise.all( + asyncOperations.map(async (r) => { + const dbRecord = await translateApiAsyncOperationToPostgresAsyncOperation(r, knex); + await asyncOperationPgModel.create(knex, dbRecord); + }) + ); +} + async function addCollections(collections) { const knex = await getKnexClient({ env: { @@ -89,11 +106,9 @@ async function addCollections(collections) { }, }); - const es = await getESClientAndIndex(); const collectionPgModel = new CollectionPgModel(); return await Promise.all( collections.map(async (c) => { - await indexer.indexCollection(es.client, c, es.index); const dbRecord = await translateApiCollectionToPostgresCollection(c); await collectionPgModel.create(knex, dbRecord); }) @@ -109,7 +124,6 @@ async function addGranules(granules) { }); const executionPgModel = new ExecutionPgModel(); - const es = await getESClientAndIndex(); return await Promise.all( granules.map(async (apiGranule) => { const newGranule = fakeGranuleFactoryV2( @@ -117,7 +131,6 @@ async function addGranules(granules) { ...apiGranule, } ); - await indexer.indexGranule(es.client, newGranule, es.index); const dbRecord = await translateApiGranuleToPostgresGranule({ dynamoRecord: newGranule, knexOrTransaction: knex, @@ -143,11 +156,9 @@ async function addProviders(providers) { }, }); - const es = await getESClientAndIndex(); const providerPgModel = new ProviderPgModel(); return await Promise.all( providers.map(async (provider) => { - await indexer.indexProvider(es.client, provider, es.index); const dbRecord = await translateApiProviderToPostgresProvider(provider); await providerPgModel.create(knex, dbRecord); }) @@ -162,12 +173,10 @@ async function addRules(rules) { }, }); - const es = await getESClientAndIndex(); const rulePgModel = new RulePgModel(); return await Promise.all( rules.map(async (r) => { const ruleRecord = await createRuleTrigger(r); - await indexer.indexRule(es.client, ruleRecord, es.index); const dbRecord = await translateApiRuleToPostgresRule(ruleRecord, knex); await rulePgModel.create(knex, dbRecord); }) @@ -182,8 +191,6 @@ async function addExecutions(executions) { }, }); - const es = await getESClientAndIndex(); - executions.sort((firstEl, secondEl) => { if (!firstEl.parentArn && !secondEl.parentArn) { return 0; @@ -199,12 +206,7 @@ async function addExecutions(executions) { const executionPgModel = new ExecutionPgModel(); const executionsIterator = async (execution) => { const dbRecord = await translateApiExecutionToPostgresExecution(execution, knex); - const [writtenPostgresDbRecord] = await executionPgModel.create(knex, dbRecord); - const apiExecutionRecord = await translatePostgresExecutionToApiExecution( - writtenPostgresDbRecord, - knex - ); - await indexer.indexExecution(es.client, apiExecutionRecord, es.index); + await executionPgModel.create(knex, dbRecord); }; await pEachSeries(executions, executionsIterator); @@ -218,11 +220,9 @@ async function addPdrs(pdrs) { }, }); - const es = await getESClientAndIndex(); const pdrPgModel = new PdrPgModel(); return await Promise.all( pdrs.map(async (p) => { - await indexer.indexPdr(es.client, p, es.index); const dbRecord = await translateApiPdrToPostgresPdr(p, knex); await pdrPgModel.create(knex, dbRecord); }) @@ -230,19 +230,24 @@ async function addPdrs(pdrs) { } async function addReconciliationReports(reconciliationReports) { - const reconciliationReportModel = new models.ReconciliationReport(); - const es = await getESClientAndIndex(); + const knex = await getKnexClient({ + env: { + ...envParams, + ...localStackConnectionEnv, + }, + }); + const reconciliationReportPgModel = new ReconciliationReportPgModel(); return await Promise.all( - reconciliationReports.map((r) => - reconciliationReportModel - .create(r) - .then((reconciliationReport) => - indexer.indexReconciliationReport(es.client, reconciliationReport, es.index))) + reconciliationReports.map(async (r) => { + const dbRecord = await translateApiReconReportToPostgresReconReport(r, knex); + await reconciliationReportPgModel.create(knex, dbRecord); + }) ); } module.exports = { resetPostgresDb, + addAsyncOperations, addProviders, addCollections, addExecutions, diff --git a/packages/api/ecs/async-operation/.nycrc.json b/packages/api/ecs/async-operation/.nycrc.json index d8341ba284b..47cc7a940cc 100644 --- a/packages/api/ecs/async-operation/.nycrc.json +++ b/packages/api/ecs/async-operation/.nycrc.json @@ -1,7 +1,7 @@ { "extends": "../../../../nyc.config.js", - "statements": 39.0, - "functions": 37.0, - "branches": 38.0, + "statements": 38.38, + "functions": 30.76, + "branches": 37.5, "lines": 38.0 } \ No newline at end of file diff --git a/packages/api/ecs/async-operation/index.js b/packages/api/ecs/async-operation/index.js index 80b1e5082fd..734e3d48678 100644 --- a/packages/api/ecs/async-operation/index.js +++ b/packages/api/ecs/async-operation/index.js @@ -20,12 +20,9 @@ const { getObject, getObjectStreamContents, } = require('@cumulus/aws-client/S3'); -const indexer = require('@cumulus/es-client/indexer'); -const { getEsClient } = require('@cumulus/es-client/search'); const { getKnexClient, AsyncOperationPgModel, - createRejectableTransaction, translatePostgresAsyncOperationToApiAsyncOperation, } = require('@cumulus/db'); @@ -165,7 +162,7 @@ function buildErrorOutput(error) { const writeAsyncOperationToPostgres = async (params) => { const { - trx, + knex, env, dbOutput, status, @@ -175,7 +172,7 @@ const writeAsyncOperationToPostgres = async (params) => { const id = env.asyncOperationId; return await asyncOperationPgModel .update( - trx, + knex, { id }, { status, @@ -186,27 +183,6 @@ const writeAsyncOperationToPostgres = async (params) => { ); }; -const writeAsyncOperationToEs = async (params) => { - const { - env, - status, - dbOutput, - updatedTime, - esClient, - } = params; - - await indexer.updateAsyncOperation( - esClient, - env.asyncOperationId, - { - status, - output: dbOutput, - updatedAt: Number(updatedTime), - }, - process.env.ES_INDEX - ); -}; - /** * Update an AsyncOperation item in Postgres * @@ -222,7 +198,6 @@ const updateAsyncOperation = async (params) => { status, output, envOverride = {}, - esClient = await getEsClient(), asyncOperationPgModel = new AsyncOperationPgModel(), } = params; @@ -234,20 +209,17 @@ const updateAsyncOperation = async (params) => { logger.info(`About to update async operation to ${JSON.stringify(status)} with output: ${dbOutput}`); const knex = await getKnexClient({ env }); - return await createRejectableTransaction(knex, async (trx) => { - const pgRecords = await writeAsyncOperationToPostgres({ - dbOutput, - env, - status, - trx, - updatedTime, - asyncOperationPgModel, - }); - const result = translatePostgresAsyncOperationToApiAsyncOperation(pgRecords[0]); - await writeAsyncOperationToEs({ env, status, dbOutput, updatedTime, esClient }); - logger.info(`Successfully updated async operation to ${JSON.stringify(status)} with output: ${JSON.stringify(dbOutput)}`); - return result; + const pgRecords = await writeAsyncOperationToPostgres({ + dbOutput, + env, + status, + knex, + updatedTime, + asyncOperationPgModel, }); + const result = translatePostgresAsyncOperationToApiAsyncOperation(pgRecords[0]); + logger.info(`Successfully updated async operation to ${JSON.stringify(status)} with output: ${JSON.stringify(dbOutput)}`); + return result; }; /** diff --git a/packages/api/ecs/async-operation/package.json b/packages/api/ecs/async-operation/package.json index a9446dc6a12..75bc16f955b 100644 --- a/packages/api/ecs/async-operation/package.json +++ b/packages/api/ecs/async-operation/package.json @@ -25,7 +25,6 @@ "@aws-sdk/client-lambda": "^3.621.0", "@cumulus/aws-client": "19.1.0", "@cumulus/db": "19.1.0", - "@cumulus/es-client": "19.1.0", "@cumulus/logger": "19.1.0", "crypto-random-string": "^3.2.0", "got": "^11.8.5", diff --git a/packages/api/ecs/async-operation/tests/test-index.js b/packages/api/ecs/async-operation/tests/test-index.js index cb6f580294f..4475bef1e9c 100644 --- a/packages/api/ecs/async-operation/tests/test-index.js +++ b/packages/api/ecs/async-operation/tests/test-index.js @@ -12,14 +12,6 @@ const { translatePostgresAsyncOperationToApiAsyncOperation, migrationDir, } = require('@cumulus/db'); -const { - indexAsyncOperation, -} = require('@cumulus/es-client/indexer'); -const { Search } = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); // eslint-disable-next-line unicorn/import-index const { updateAsyncOperation } = require('../index'); @@ -32,15 +24,6 @@ test.before(async (t) => { t.context.testKnexAdmin = knexAdmin; t.context.asyncOperationPgModel = new AsyncOperationPgModel(); - - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esAsyncOperationsClient = new Search( - {}, - 'asyncOperation', - t.context.esIndex - ); }); test.beforeEach(async (t) => { @@ -49,18 +32,13 @@ test.beforeEach(async (t) => { t.context.testAsyncOperation = { id: t.context.asyncOperationId, description: 'test description', - operationType: 'ES Index', + operationType: 'Reconciliation Report', status: 'RUNNING', createdAt: Date.now(), }; t.context.testAsyncOperationPgRecord = translateApiAsyncOperationToPostgresAsyncOperation( t.context.testAsyncOperation ); - await indexAsyncOperation( - t.context.esClient, - t.context.testAsyncOperation, - t.context.esIndex - ); await t.context.asyncOperationPgModel.create( t.context.testKnex, t.context.testAsyncOperationPgRecord @@ -73,10 +51,9 @@ test.after.always(async (t) => { knexAdmin: t.context.testKnexAdmin, testDbName, }); - await cleanupTestIndex(t.context); }); -test('updateAsyncOperation updates databases as expected', async (t) => { +test('updateAsyncOperation updates database as expected', async (t) => { const status = 'SUCCEEDED'; const output = { foo: 'bar' }; const updateTime = (Number(Date.now())).toString(); @@ -107,21 +84,9 @@ test('updateAsyncOperation updates databases as expected', async (t) => { output, updated_at: new Date(Number(updateTime)), }); - - const asyncOpEsRecord = await t.context.esAsyncOperationsClient.get( - t.context.testAsyncOperation.id - ); - t.deepEqual(asyncOpEsRecord, { - ...t.context.testAsyncOperation, - _id: asyncOpEsRecord._id, - timestamp: asyncOpEsRecord.timestamp, - status, - output: JSON.stringify(output), - updatedAt: Number(updateTime), - }); }); -test('updateAsyncOperation updates records correctly when output is undefined', async (t) => { +test('updateAsyncOperation updates record correctly when output is undefined', async (t) => { const status = 'SUCCEEDED'; const output = undefined; const updateTime = (Number(Date.now())).toString(); @@ -154,7 +119,7 @@ test('updateAsyncOperation updates records correctly when output is undefined', }); }); -test('updateAsyncOperation updates databases with correct timestamps', async (t) => { +test('updateAsyncOperation updates database with correct timestamps', async (t) => { const status = 'SUCCEEDED'; const output = { foo: 'bar' }; const updateTime = (Number(Date.now())).toString(); @@ -179,95 +144,3 @@ test('updateAsyncOperation updates databases with correct timestamps', async (t) ); t.is(asyncOperationPgRecord.updated_at.getTime().toString(), updateTime); }); - -test('updateAsyncOperation does not update PostgreSQL if write to Elasticsearch fails', async (t) => { - const status = 'SUCCEEDED'; - const output = { foo: cryptoRandomString({ length: 5 }) }; - const updateTime = (Number(Date.now())).toString(); - - const fakeEsClient = { - client: { - update: () => { - throw new Error('ES fail'); - }, - }, - }; - - await t.throwsAsync( - updateAsyncOperation({ - status, - output, - envOverride: { - asyncOperationId: t.context.asyncOperationId, - ...localStackConnectionEnv, - PG_DATABASE: testDbName, - updateTime, - }, - esClient: fakeEsClient, - }), - { message: 'ES fail' } - ); - - const asyncOperationPgRecord = await t.context.asyncOperationPgModel - .get( - t.context.testKnex, - { - id: t.context.asyncOperationId, - } - ); - t.like(asyncOperationPgRecord, t.context.testAsyncOperationPgRecord); - - const asyncOpEsRecord = await t.context.esAsyncOperationsClient.get( - t.context.testAsyncOperation.id - ); - t.deepEqual(asyncOpEsRecord, { - ...t.context.testAsyncOperation, - _id: asyncOpEsRecord._id, - timestamp: asyncOpEsRecord.timestamp, - }); -}); - -test('updateAsyncOperation does not update Elasticsearch if write to PostgreSQL fails', async (t) => { - const status = 'SUCCEEDED'; - const output = { foo: cryptoRandomString({ length: 5 }) }; - const updateTime = (Number(Date.now())).toString(); - - const fakePgModel = { - update: () => { - throw new Error('PG fail'); - }, - }; - - await t.throwsAsync( - updateAsyncOperation({ - status, - output, - envOverride: { - asyncOperationId: t.context.asyncOperationId, - ...localStackConnectionEnv, - PG_DATABASE: testDbName, - updateTime, - }, - asyncOperationPgModel: fakePgModel, - }), - { message: 'PG fail' } - ); - - const asyncOperationPgRecord = await t.context.asyncOperationPgModel - .get( - t.context.testKnex, - { - id: t.context.asyncOperationId, - } - ); - t.like(asyncOperationPgRecord, t.context.testAsyncOperationPgRecord); - - const asyncOpEsRecord = await t.context.esAsyncOperationsClient.get( - t.context.testAsyncOperation.id - ); - t.deepEqual(asyncOpEsRecord, { - ...t.context.testAsyncOperation, - _id: asyncOpEsRecord._id, - timestamp: asyncOpEsRecord.timestamp, - }); -}); diff --git a/packages/api/endpoints/async-operations.js b/packages/api/endpoints/async-operations.js index 15de1821a4d..c54c3f83ac1 100644 --- a/packages/api/endpoints/async-operations.js +++ b/packages/api/endpoints/async-operations.js @@ -1,3 +1,5 @@ +//@ts-check + 'use strict'; const router = require('express-promise-router')(); @@ -8,20 +10,15 @@ const { getKnexClient, translateApiAsyncOperationToPostgresAsyncOperation, translatePostgresAsyncOperationToApiAsyncOperation, - createRejectableTransaction, } = require('@cumulus/db'); const { RecordDoesNotExist, ValidationError, } = require('@cumulus/errors'); -const { - indexAsyncOperation, -} = require('@cumulus/es-client/indexer'); const Logger = require('@cumulus/logger'); +const { AsyncOperationSearch } = require('@cumulus/db'); -const { Search, getEsClient } = require('@cumulus/es-client/search'); -const { deleteAsyncOperation } = require('@cumulus/es-client/indexer'); const { isBadRequestError } = require('../lib/errors'); const { recordIsValid } = require('../lib/schema'); @@ -30,13 +27,9 @@ const asyncSchema = require('../lib/schemas').asyncOperation; const logger = new Logger({ sender: '@cumulus/api/asyncOperations' }); async function list(req, res) { - const search = new Search( - { queryStringParameters: req.query }, - 'asyncOperation', - process.env.ES_INDEX - ); + const dbSearch = new AsyncOperationSearch({ queryStringParameters: req.query }); - const response = await search.query(); + const response = await dbSearch.query(); return res.send(response); } @@ -74,16 +67,9 @@ async function del(req, res) { const { asyncOperationPgModel = new AsyncOperationPgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const { id } = req.params || {}; - const esAsyncOperationsClient = new Search( - {}, - 'asyncOperation', - process.env.ES_INDEX - ); - if (!id) { return res.boom.badRequest('id parameter is missing'); } @@ -92,26 +78,13 @@ async function del(req, res) { await asyncOperationPgModel.get(knex, { id }); } catch (error) { if (error instanceof RecordDoesNotExist) { - if (!(await esAsyncOperationsClient.exists(id))) { - logger.info('Async Operation does not exist in Elasticsearch and PostgreSQL'); - return res.boom.notFound('No record found'); - } - logger.info('Async Operation does not exist in PostgreSQL, it only exists in Elasticsearch. Proceeding with deletion'); - } else { - throw error; + logger.info('Async Operation does not exist PostgreSQL'); + return res.boom.notFound('No record found'); } + return res.boom.badImplementation(JSON.stringify(error)); } - await createRejectableTransaction(knex, async (trx) => { - await asyncOperationPgModel.delete(trx, { id }); - await deleteAsyncOperation({ - esClient, - id, - index: process.env.ES_INDEX, - ignore: [404], - }); - }); - + await asyncOperationPgModel.delete(knex, { id }); return res.send({ message: 'Record deleted' }); } @@ -126,7 +99,6 @@ async function post(req, res) { const { asyncOperationPgModel = new AsyncOperationPgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const apiAsyncOperation = req.body; @@ -144,12 +116,8 @@ async function post(req, res) { } const dbRecord = translateApiAsyncOperationToPostgresAsyncOperation(apiAsyncOperation); logger.debug(`Attempting to create async operation ${dbRecord.id}`); - let apiDbRecord; - await createRejectableTransaction(knex, async (trx) => { - const pgRecord = await asyncOperationPgModel.create(trx, dbRecord, ['*']); - apiDbRecord = await translatePostgresAsyncOperationToApiAsyncOperation(pgRecord[0]); - await indexAsyncOperation(esClient, apiDbRecord, process.env.ES_INDEX); - }); + const pgRecord = await asyncOperationPgModel.create(knex, dbRecord, ['*']); + const apiDbRecord = translatePostgresAsyncOperationToApiAsyncOperation(pgRecord[0]); logger.info(`Successfully created async operation ${apiDbRecord.id}:`); return res.send({ message: 'Record saved', diff --git a/packages/api/endpoints/elasticsearch.js b/packages/api/endpoints/elasticsearch.js deleted file mode 100644 index b09bb81572e..00000000000 --- a/packages/api/endpoints/elasticsearch.js +++ /dev/null @@ -1,233 +0,0 @@ -'use strict'; - -const router = require('express-promise-router')(); -const { v4: uuidv4 } = require('uuid'); - -const log = require('@cumulus/common/log'); -const { IndexExistsError } = require('@cumulus/errors'); -const { defaultIndexAlias, getEsClient } = require('@cumulus/es-client/search'); -const { createIndex } = require('@cumulus/es-client/indexer'); - -const { asyncOperationEndpointErrorHandler } = require('../app/middleware'); -const { getFunctionNameFromRequestContext } = require('../lib/request'); -const startAsyncOperation = require('../lib/startAsyncOperation'); - -// const snapshotRepoName = 'cumulus-es-snapshots'; - -function timestampedIndexName() { - const date = new Date(); - return `cumulus-${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`; -} - -function createEsSnapshot(req, res) { - return res.boom.badRequest('Functionality not yet implemented'); -} - -async function reindex(req, res) { - let sourceIndex = req.body.sourceIndex; - let destIndex = req.body.destIndex; - const aliasName = req.body.aliasName || defaultIndexAlias; - - const esClient = await getEsClient(); - - if (!sourceIndex) { - const alias = await esClient.client.indices.getAlias({ - name: aliasName, - }).then((response) => response.body); - - // alias keys = index name - const indices = Object.keys(alias); - - if (indices.length > 1) { - // We don't know which index to use as the source, throw error - return res.boom.badRequest(`Multiple indices found for alias ${aliasName}. Specify source index as one of [${indices.sort().join(', ')}].`); - } - - sourceIndex = indices[0]; - } else { - const sourceExists = await esClient.client.indices.exists({ index: sourceIndex }) - .then((response) => response.body); - - if (!sourceExists) { - return res.boom.badRequest(`Source index ${sourceIndex} does not exist.`); - } - } - - if (!destIndex) { - destIndex = timestampedIndexName(); - } - - if (sourceIndex === destIndex) { - return res.boom.badRequest(`source index(${sourceIndex}) and destination index(${destIndex}) must be different.`); - } - - const destExists = await esClient.client.indices.exists({ index: destIndex }) - .then((response) => response.body); - - if (!destExists) { - try { - await createIndex(esClient, destIndex); - log.info(`Created destination index ${destIndex}.`); - } catch (error) { - return res.boom.badRequest(`Error creating index ${destIndex}: ${error.message}`); - } - } - - // reindex - esClient.client.reindex({ - body: { - source: { index: sourceIndex }, - dest: { index: destIndex }, - }, - }); - - const message = `Reindexing to ${destIndex} from ${sourceIndex}. Check the reindex-status endpoint for status.`; - - return res.status(200).send({ message }); -} - -async function reindexStatus(req, res) { - const esClient = await getEsClient(); - - const reindexTaskStatus = await esClient.client.tasks.list({ actions: ['*reindex'] }) - .then((response) => response.body); - - await esClient.client.indices.refresh(); - - const indexStatus = await esClient.client.indices.stats({ - metric: 'docs', - }).then((response) => response.body); - - const status = { - reindexStatus: reindexTaskStatus, - indexStatus, - }; - - return res.send(status); -} - -async function changeIndex(req, res) { - const deleteSource = req.body.deleteSource; - const aliasName = req.body.aliasName || defaultIndexAlias; - const currentIndex = req.body.currentIndex; - const newIndex = req.body.newIndex; - - const esClient = await getEsClient(); - - if (!currentIndex || !newIndex) { - return res.boom.badRequest('Please explicity specify a current and new index.'); - } - - if (currentIndex === newIndex) { - return res.boom.badRequest('The current index cannot be the same as the new index.'); - } - - const currentExists = await esClient.client.indices.exists({ index: currentIndex }) - .then((response) => response.body); - - if (!currentExists) { - return res.boom.badRequest(`Current index ${currentIndex} does not exist.`); - } - - const destExists = await esClient.client.indices.exists({ index: newIndex }) - .then((response) => response.body); - - if (!destExists) { - try { - await createIndex(esClient, newIndex); - log.info(`Created destination index ${newIndex}.`); - } catch (error) { - return res.boom.badRequest(`Error creating index ${newIndex}: ${error.message}`); - } - } - - try { - await esClient.client.indices.updateAliases({ - body: { - actions: [ - { remove: { index: currentIndex, alias: aliasName } }, - { add: { index: newIndex, alias: aliasName } }, - ], - }, - }); - - log.info(`Removed alias ${aliasName} from index ${currentIndex} and added alias to ${newIndex}`); - } catch (error) { - return res.boom.badRequest( - `Error removing alias ${aliasName} from index ${currentIndex} and adding alias to ${newIndex}: ${error}` - ); - } - - let message = `Change index success - alias ${aliasName} now pointing to ${newIndex}`; - - if (deleteSource) { - await esClient.client.indices.delete({ index: currentIndex }); - log.info(`Deleted index ${currentIndex}`); - message = `${message} and index ${currentIndex} deleted`; - } - - return res.send({ message }); -} - -async function indicesStatus(req, res) { - const esClient = await getEsClient(); - - return res.send(await esClient.client.cat.indices({})); -} - -async function indexFromDatabase(req, res) { - const esClient = await getEsClient(); - const indexName = req.body.indexName || timestampedIndexName(); - const { postgresResultPageSize, postgresConnectionPoolSize, esRequestConcurrency } = req.body; - - await createIndex(esClient, indexName) - .catch((error) => { - if (!(error instanceof IndexExistsError)) throw error; - }); - - const asyncOperationId = uuidv4(); - const asyncOperationEvent = { - asyncOperationId, - callerLambdaName: getFunctionNameFromRequestContext(req), - lambdaName: process.env.IndexFromDatabaseLambda, - description: 'Elasticsearch index from database', - operationType: 'ES Index', - payload: { - indexName, - reconciliationReportsTable: process.env.ReconciliationReportsTable, - esHost: process.env.ES_HOST, - esRequestConcurrency: esRequestConcurrency || process.env.ES_CONCURRENCY, - postgresResultPageSize, - postgresConnectionPoolSize, - }, - }; - - log.debug(`About to invoke lambda to start async operation ${asyncOperationId}`); - await startAsyncOperation.invokeStartAsyncOperationLambda(asyncOperationEvent); - return res.send({ message: `Indexing database to ${indexName}. Operation id: ${asyncOperationId}` }); -} - -async function getCurrentIndex(req, res) { - const esClient = await getEsClient(); - const alias = req.params.alias || defaultIndexAlias; - - const aliasIndices = await esClient.client.indices.getAlias({ name: alias }) - .then((response) => response.body); - - return res.send(Object.keys(aliasIndices)); -} - -// express routes -router.put('/create-snapshot', createEsSnapshot); -router.post('/reindex', reindex); -router.get('/reindex-status', reindexStatus); -router.post('/change-index', changeIndex); -router.post('/index-from-database', indexFromDatabase, asyncOperationEndpointErrorHandler); -router.get('/indices-status', indicesStatus); -router.get('/current-index/:alias', getCurrentIndex); -router.get('/current-index', getCurrentIndex); - -module.exports = { - indexFromDatabase, - router, -}; diff --git a/packages/api/endpoints/pdrs.js b/packages/api/endpoints/pdrs.js index 6023cbec4ef..5725039f3d5 100644 --- a/packages/api/endpoints/pdrs.js +++ b/packages/api/endpoints/pdrs.js @@ -1,3 +1,5 @@ +//@ts-check + 'use strict'; const router = require('express-promise-router')(); @@ -9,8 +11,7 @@ const { createRejectableTransaction, } = require('@cumulus/db'); const { RecordDoesNotExist } = require('@cumulus/errors'); -const { indexPdr, deletePdr } = require('@cumulus/es-client/indexer'); -const { Search, getEsClient } = require('@cumulus/es-client/search'); +const { PdrSearch } = require('@cumulus/db'); const Logger = require('@cumulus/logger'); const log = new Logger({ sender: '@cumulus/api/pdrs' }); @@ -23,12 +24,8 @@ const log = new Logger({ sender: '@cumulus/api/pdrs' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { - const search = new Search( - { queryStringParameters: req.query }, - 'pdr', - process.env.ES_INDEX - ); - const result = await search.query(); + const dbSearch = new PdrSearch({ queryStringParameters: req.query }); + const result = await dbSearch.query(); return res.send(result); } @@ -57,8 +54,6 @@ async function get(req, res) { } } -const isRecordDoesNotExistError = (e) => e.message.includes('RecordDoesNotExist'); - /** * delete a given PDR * @@ -70,63 +65,23 @@ async function del(req, res) { const { pdrPgModel = new PdrPgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), s3Utils = S3UtilsLib, } = req.testContext || {}; const pdrName = req.params.pdrName; const pdrS3Key = `${process.env.stackName}/pdrs/${pdrName}`; - const esPdrsClient = new Search( - {}, - 'pdr', - process.env.ES_INDEX - ); try { - await pdrPgModel.get(knex, { name: pdrName }); - } catch (error) { - if (error instanceof RecordDoesNotExist) { - if (!(await esPdrsClient.exists(pdrName))) { - log.info('PDR does not exist in Elasticsearch'); + await createRejectableTransaction(knex, async (trx) => { + const deleteResultsCount = await pdrPgModel.delete(trx, { name: pdrName }); + if (deleteResultsCount === 0) { return res.boom.notFound('No record found'); } - log.info('PDR does not exist in PostgreSQL, it only exists in Elasticsearch'); - } else { - throw error; - } - } - - const esPdrClient = new Search( - {}, - 'pdr', - process.env.ES_INDEX - ); - const esPdrRecord = await esPdrClient.get(pdrName).catch(log.info); - - try { - let esPdrDeleted = false; - try { - await createRejectableTransaction(knex, async (trx) => { - await pdrPgModel.delete(trx, { name: pdrName }); - await deletePdr({ - esClient, - name: pdrName, - index: process.env.ES_INDEX, - ignore: [404], - }); - esPdrDeleted = true; - await s3Utils.deleteS3Object(process.env.system_bucket, pdrS3Key); - }); - } catch (innerError) { - if (esPdrDeleted && esPdrRecord) { - delete esPdrRecord._id; - await indexPdr(esClient, esPdrRecord, process.env.ES_INDEX); - } - throw innerError; - } + return await s3Utils.deleteS3Object(process.env.system_bucket, pdrS3Key); + }); } catch (error) { log.debug(`Failed to delete PDR with name ${pdrName}. Error ${JSON.stringify(error)}.`); - if (!isRecordDoesNotExistError(error)) throw error; + throw error; } return res.send({ detail: 'Record deleted' }); } diff --git a/packages/api/endpoints/providers.js b/packages/api/endpoints/providers.js index 7c9dd3cdec5..8f040980191 100644 --- a/packages/api/endpoints/providers.js +++ b/packages/api/endpoints/providers.js @@ -10,14 +10,13 @@ const { translateApiProviderToPostgresProvider, translatePostgresProviderToApiProvider, validateProviderHost, + ProviderSearch, } = require('@cumulus/db'); const { RecordDoesNotExist, ValidationError, } = require('@cumulus/errors'); const Logger = require('@cumulus/logger'); -const { getEsClient, Search } = require('@cumulus/es-client/search'); -const { indexProvider, deleteProvider } = require('@cumulus/es-client/indexer'); const { removeNilProperties } = require('@cumulus/common/util'); const { isBadRequestError } = require('../lib/errors'); @@ -31,14 +30,11 @@ const log = new Logger({ sender: '@cumulus/api/providers' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { - const search = new Search( - { queryStringParameters: req.query }, - 'provider', - process.env.ES_INDEX + const dbSearch = new ProviderSearch( + { queryStringParameters: req.query } ); - - const response = await search.query(); - return res.send(response); + const result = await dbSearch.query(); + return res.send(result); } /** @@ -75,7 +71,6 @@ async function post(req, res) { const { providerPgModel = new ProviderPgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const apiProvider = req.body; @@ -97,7 +92,6 @@ async function post(req, res) { await createRejectableTransaction(knex, async (trx) => { const [updatedPostgresProvider] = await providerPgModel.create(trx, postgresProvider, '*'); record = translatePostgresProviderToApiProvider(updatedPostgresProvider); - await indexProvider(esClient, record, process.env.ES_INDEX); }); return res.send({ record, message: 'Record saved' }); } catch (error) { @@ -125,7 +119,6 @@ async function put(req, res) { const { providerPgModel = new ProviderPgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const { params: { id }, body } = req; @@ -160,7 +153,6 @@ async function put(req, res) { await createRejectableTransaction(knex, async (trx) => { const [updatedPostgresProvider] = await providerPgModel.upsert(trx, postgresProvider); record = translatePostgresProviderToApiProvider(updatedPostgresProvider); - await indexProvider(esClient, record, process.env.ES_INDEX); }); return res.send(record); @@ -177,39 +169,23 @@ async function del(req, res) { const { providerPgModel = new ProviderPgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const { id } = req.params; - const esProvidersClient = new Search( - {}, - 'provider', - process.env.ES_INDEX - ); try { await providerPgModel.get(knex, { name: id }); } catch (error) { if (error instanceof RecordDoesNotExist) { - if (!(await esProvidersClient.exists(id))) { - log.info('Provider does not exist in Elasticsearch and PostgreSQL'); - return res.boom.notFound('No record found'); - } - log.info('Provider does not exist in PostgreSQL, it only exists in Elasticsearch. Proceeding with deletion'); - } else { - throw error; + log.info('Provider does not exist in PostgreSQL'); + return res.boom.notFound('No record found'); } + throw error; } try { await createRejectableTransaction(knex, async (trx) => { await providerPgModel.delete(trx, { name: id }); - await deleteProvider({ - esClient, - id, - index: process.env.ES_INDEX, - ignore: [404], - }); }); log.debug(`deleted provider ${id}`); return res.send({ message: 'Record deleted' }); diff --git a/packages/api/endpoints/reconciliation-reports.js b/packages/api/endpoints/reconciliation-reports.js index 98069825aab..68467065668 100644 --- a/packages/api/endpoints/reconciliation-reports.js +++ b/packages/api/endpoints/reconciliation-reports.js @@ -1,3 +1,5 @@ +//@ts-check + 'use strict'; const router = require('express-promise-router')(); @@ -6,7 +8,7 @@ const { deleteS3Object, fileExists, getObjectSize, - getS3Object, + getObject, parseS3Uri, buildS3Uri, getObjectStreamContents, @@ -14,13 +16,16 @@ const { const S3ObjectStore = require('@cumulus/aws-client/S3ObjectStore'); const { s3 } = require('@cumulus/aws-client/services'); -const { inTestMode } = require('@cumulus/common/test-utils'); const { RecordDoesNotExist } = require('@cumulus/errors'); const Logger = require('@cumulus/logger'); -const { Search, getEsClient } = require('@cumulus/es-client/search'); -const indexer = require('@cumulus/es-client/indexer'); -const models = require('../models'); +const { ReconciliationReportSearch } = require('@cumulus/db'); + +const { + ReconciliationReportPgModel, + createRejectableTransaction, + getKnexClient, +} = require('@cumulus/db'); const { normalizeEvent } = require('../lib/reconciliationReport/normalizeEvent'); const startAsyncOperation = require('../lib/startAsyncOperation'); const { asyncOperationEndpointErrorHandler } = require('../app/middleware'); @@ -29,6 +34,11 @@ const { getFunctionNameFromRequestContext } = require('../lib/request'); const logger = new Logger({ sender: '@cumulus/api' }); const maxResponsePayloadSizeBytes = 6 * 1000 * 1000; +/** +* @typedef {import('../lib/types').NormalizedRecReportParams} NormalizedRecReportParams +* @typedef {import('../lib/types').RecReportParams} RecReportParams +*/ + /** * List all reconciliation reports * @@ -37,14 +47,11 @@ const maxResponsePayloadSizeBytes = 6 * 1000 * 1000; * @returns {Promise} the promise of express response object */ async function listReports(req, res) { - const search = new Search( - { queryStringParameters: req.query }, - 'reconciliationReport', - process.env.ES_INDEX + const dbSearch = new ReconciliationReportSearch( + { queryStringParameters: req.query } ); - - const response = await search.query(); - return res.send(response); + const result = await dbSearch.query(); + return res.send(result); } /** @@ -56,10 +63,14 @@ async function listReports(req, res) { */ async function getReport(req, res) { const name = req.params.name; - const reconciliationReportModel = new models.ReconciliationReport(); try { - const result = await reconciliationReportModel.get({ name }); + const reconciliationReportPgModel = new ReconciliationReportPgModel(); + const knex = await getKnexClient(); + const result = await reconciliationReportPgModel.get(knex, { name }); + if (!result.location) { + return res.boom.badRequest('The reconciliation report record does not contain a location.'); + } const { Bucket, Key } = parseS3Uri(result.location); const reportExists = await fileExists(Bucket, Key); if (!reportExists) { @@ -77,20 +88,22 @@ async function getReport(req, res) { ); if (Key.endsWith('.json') || Key.endsWith('.csv')) { - const reportSize = await getObjectSize({ s3: s3(), bucket: Bucket, key: Key }); + const reportSize = await getObjectSize({ s3: s3(), bucket: Bucket, key: Key }) ?? 0; // estimated payload size, add extra const estimatedPayloadSize = presignedS3Url.length + reportSize + 50; - if ( - estimatedPayloadSize - > (process.env.maxResponsePayloadSizeBytes || maxResponsePayloadSizeBytes) + if (estimatedPayloadSize > + Number(process.env.maxResponsePayloadSizeBytes || maxResponsePayloadSizeBytes) ) { res.json({ presignedS3Url, data: `Error: Report ${name} exceeded maximum allowed payload size`, }); } else { - const file = await getS3Object(Bucket, Key); + const file = await getObject(s3(), { Bucket, Key }); logger.debug(`Sending json file with contentLength ${file.ContentLength}`); + if (!file.Body) { + return res.boom.badRequest('Report file does not have a body.'); + } const fileBody = await getObjectStreamContents(file.Body); return res.json({ presignedS3Url, @@ -98,14 +111,15 @@ async function getReport(req, res) { }); } } - logger.debug('reconciliation report getReport received an unhandled report type.'); + logger.debug('Reconciliation report getReport received an unhandled report type.'); } catch (error) { if (error instanceof RecordDoesNotExist) { return res.boom.notFound(`No record found for ${name}`); } throw error; } - return res.boom.badImplementation('reconciliation report getReport failed in an indeterminate manner.'); + + return res.boom.badImplementation('Reconciliation report getReport failed in an indeterminate manner.'); } /** @@ -117,25 +131,30 @@ async function getReport(req, res) { */ async function deleteReport(req, res) { const name = req.params.name; - const reconciliationReportModel = new models.ReconciliationReport(); - const record = await reconciliationReportModel.get({ name }); + let record; - const { Bucket, Key } = parseS3Uri(record.location); - if (await fileExists(Bucket, Key)) { - await deleteS3Object(Bucket, Key); + const reconciliationReportPgModel = new ReconciliationReportPgModel(); + const knex = await getKnexClient(); + try { + record = await reconciliationReportPgModel.get(knex, { name }); + } catch (error) { + if (error instanceof RecordDoesNotExist) { + return res.boom.notFound(`No record found for ${name}`); + } + throw error; } - await reconciliationReportModel.delete({ name }); - - if (inTestMode()) { - const esClient = await getEsClient(process.env.ES_HOST); - await indexer.deleteRecord({ - esClient, - id: name, - type: 'reconciliationReport', - index: process.env.ES_INDEX, - ignore: [404], - }); + + if (!record.location) { + return res.boom.badRequest('The reconciliation report record does not contain a location!'); } + const { Bucket, Key } = parseS3Uri(record.location); + + await createRejectableTransaction(knex, async () => { + if (await fileExists(Bucket, Key)) { + await deleteS3Object(Bucket, Key); + } + await reconciliationReportPgModel.delete(knex, { name }); + }); return res.send({ message: 'Report deleted' }); } @@ -144,10 +163,12 @@ async function deleteReport(req, res) { * Creates a new report * * @param {Object} req - express request object + * @param {RecReportParams} req.body * @param {Object} res - express response object * @returns {Promise} the promise of express response object */ async function createReport(req, res) { + /** @type NormalizedRecReportParams */ let validatedInput; try { validatedInput = normalizeEvent(req.body); diff --git a/packages/api/endpoints/rules.js b/packages/api/endpoints/rules.js index 38897c720da..0097ca6936b 100644 --- a/packages/api/endpoints/rules.js +++ b/packages/api/endpoints/rules.js @@ -14,11 +14,10 @@ const { getKnexClient, isCollisionError, RulePgModel, + RuleSearch, translateApiRuleToPostgresRuleRaw, translatePostgresRuleToApiRule, } = require('@cumulus/db'); -const { Search, getEsClient } = require('@cumulus/es-client/search'); -const { indexRule, deleteRule } = require('@cumulus/es-client/indexer'); const { requireApiVersion, @@ -47,12 +46,11 @@ const log = new Logger({ sender: '@cumulus/api/rules' }); * @returns {Promise} the promise of express response object */ async function list(req, res) { - const search = new Search( - { queryStringParameters: req.query }, - 'rule', - process.env.ES_INDEX + const dbSearch = new RuleSearch( + { queryStringParameters: req.query } ); - const response = await search.query(); + + const response = await dbSearch.query(); return res.send(response); } @@ -93,7 +91,6 @@ async function post(req, res) { const { rulePgModel = new RulePgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; let record; @@ -116,7 +113,6 @@ async function post(req, res) { await createRejectableTransaction(knex, async (trx) => { const [pgRecord] = await rulePgModel.create(trx, postgresRule); record = await translatePostgresRuleToApiRule(pgRecord, knex); - await indexRule(esClient, record, process.env.ES_INDEX); }); } catch (innerError) { if (isCollisionError(innerError)) { @@ -143,7 +139,6 @@ async function post(req, res) { * @param {object} params.apiRule - updated API rule * @param {object} params.rulePgModel - @cumulus/db compatible rule module instance * @param {object} params.knex - Knex object - * @param {object} params.esClient - Elasticsearch client * @returns {Promise} - promise of an express response object. */ async function patchRule(params) { @@ -153,7 +148,6 @@ async function patchRule(params) { apiRule, rulePgModel = new RulePgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = params; log.debug(`rules.patchRule oldApiRule: ${JSON.stringify(oldApiRule)}, apiRule: ${JSON.stringify(apiRule)}`); @@ -172,7 +166,6 @@ async function patchRule(params) { const [pgRule] = await rulePgModel.upsert(trx, apiPgRule); log.debug(`rules.patchRule pgRule: ${JSON.stringify(pgRule)}`); translatedRule = await translatePostgresRuleToApiRule(pgRule, knex); - await indexRule(esClient, translatedRule, process.env.ES_INDEX); }); log.info(`rules.patchRule translatedRule: ${JSON.stringify(translatedRule)}`); @@ -198,7 +191,6 @@ async function patch(req, res) { const { rulePgModel = new RulePgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const { params: { name }, body } = req; @@ -216,7 +208,7 @@ async function patch(req, res) { apiRule.createdAt = oldApiRule.createdAt; apiRule = merge(cloneDeep(oldApiRule), apiRule); - return await patchRule({ res, oldApiRule, apiRule, knex, esClient, rulePgModel }); + return await patchRule({ res, oldApiRule, apiRule, knex, rulePgModel }); } catch (error) { log.error('Unexpected error when updating rule:', error); if (error instanceof RecordDoesNotExist) { @@ -242,7 +234,6 @@ async function put(req, res) { const { rulePgModel = new RulePgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const { params: { name }, body } = req; @@ -272,7 +263,7 @@ async function put(req, res) { apiRule.createdAt = oldApiRule.createdAt; - return await patchRule({ res, oldApiRule, apiRule, knex, esClient, rulePgModel }); + return await patchRule({ res, oldApiRule, apiRule, knex, rulePgModel }); } catch (error) { log.error('Unexpected error when updating rule:', error); if (error instanceof RecordDoesNotExist) { @@ -293,15 +284,10 @@ async function del(req, res) { const { rulePgModel = new RulePgModel(), knex = await getKnexClient(), - esClient = await getEsClient(), } = req.testContext || {}; const name = (req.params.name || '').replace(/%20/g, ' '); - const esRulesClient = new Search( - {}, - 'rule', - process.env.ES_INDEX - ); + let rule; let apiRule; @@ -309,26 +295,14 @@ async function del(req, res) { rule = await rulePgModel.get(knex, { name }); apiRule = await translatePostgresRuleToApiRule(rule, knex); } catch (error) { - // If rule doesn't exist in PG or ES, return not found if (error instanceof RecordDoesNotExist) { - if (!(await esRulesClient.exists(name))) { - log.info('Rule does not exist in Elasticsearch and PostgreSQL'); - return res.boom.notFound('No record found'); - } - log.info('Rule does not exist in PostgreSQL, it only exists in Elasticsearch. Proceeding with deletion'); - } else { - throw error; + return res.boom.notFound('No record found'); } + throw error; } await createRejectableTransaction(knex, async (trx) => { await rulePgModel.delete(trx, { name }); - await deleteRule({ - esClient, - name, - index: process.env.ES_INDEX, - ignore: [404], - }); if (rule) await deleteRuleResources(knex, apiRule); }); diff --git a/packages/api/endpoints/stats.js b/packages/api/endpoints/stats.js index 8a27b380246..1ee9a521a98 100644 --- a/packages/api/endpoints/stats.js +++ b/packages/api/endpoints/stats.js @@ -21,6 +21,7 @@ function getType(req) { logs: 'logs', providers: 'provider', executions: 'execution', + reconciliationReports: 'reconciliationReport', }; const typeRequested = get(req, 'params.type') || get(req, 'query.type'); diff --git a/packages/api/lambdas/bootstrap.js b/packages/api/lambdas/bootstrap.js deleted file mode 100644 index 5a038058070..00000000000 --- a/packages/api/lambdas/bootstrap.js +++ /dev/null @@ -1,35 +0,0 @@ -/* this module is intended to be used for bootstraping - * the cloudformation deployment of a DAAC. - * - * It helps: - * - adding ElasticSearch index mapping when a new index is created - */ - -'use strict'; - -const log = require('@cumulus/common/log'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); - -/** - * Bootstrap Elasticsearch indexes - * - * @param {Object} event - AWS Lambda event input - * @returns {Promise} a Terraform Lambda invocation response - */ -const handler = async ({ elasticsearchHostname, removeAliasConflict, testContext = {} }) => { - const bootstrapFunction = testContext.bootstrapFunction || bootstrapElasticSearch; - try { - await bootstrapFunction({ - host: elasticsearchHostname, - removeAliasConflict, - }); - return { Status: 'SUCCESS', Data: {} }; - } catch (error) { - log.error(error); - throw error; - } -}; - -module.exports = { - handler, -}; diff --git a/packages/api/lambdas/bulk-operation.js b/packages/api/lambdas/bulk-operation.js index 2b19194a705..763c97e685a 100644 --- a/packages/api/lambdas/bulk-operation.js +++ b/packages/api/lambdas/bulk-operation.js @@ -93,8 +93,8 @@ async function applyWorkflowToGranules({ * Defaults to `concurrency` * @param {number} [payload.concurrency] * granule concurrency for the bulk deletion operation. Defaults to 10 - * @param {Object} [payload.query] - Optional parameter of query to send to ES - * @param {string} [payload.index] - Optional parameter of ES index to query. + * @param {Object} [payload.query] - Optional parameter of query to send to ES (Cloud Metrics) + * @param {string} [payload.index] - Optional parameter of ES index to query (Cloud Metrics). * Must exist if payload.query exists. * @param {Object} [payload.granules] - Optional list of granule unique IDs to bulk operate on * e.g. { granuleId: xxx, collectionID: xxx } @@ -178,8 +178,8 @@ async function bulkGranuleDelete( * @param {string} payload.workflowName - name of the workflow that will be applied to each granule. * @param {Object} [payload.meta] - Optional meta to add to workflow input * @param {string} [payload.queueUrl] - Optional name of queue that will be used to start workflows - * @param {Object} [payload.query] - Optional parameter of query to send to ES - * @param {string} [payload.index] - Optional parameter of ES index to query. + * @param {Object} [payload.query] - Optional parameter of query to send to ES (Cloud Metrics) + * @param {string} [payload.index] - Optional parameter of ES index to query (Cloud Metrics). * Must exist if payload.query exists. * @param {Object} [payload.granules] - Optional list of granule unique IDs to bulk operate on * e.g. { granuleId: xxx, collectionID: xxx } diff --git a/packages/api/lambdas/cleanExecutions.js b/packages/api/lambdas/cleanExecutions.js index 9699b0ce9d7..01051970ff6 100644 --- a/packages/api/lambdas/cleanExecutions.js +++ b/packages/api/lambdas/cleanExecutions.js @@ -2,107 +2,17 @@ 'use strict'; -const { getEsClient, esConfig } = require('@cumulus/es-client/search'); -const moment = require('moment'); +/** + * This lambda has a dummy handler because it needs to be rewritten for PG instead of running + * in ElasticSearch. This will be done in CUMULUS-3982. + * When this is being rewritten, redo the test file also. + */ + const Logger = require('@cumulus/logger'); -const { sleep } = require('@cumulus/common'); const log = new Logger({ sender: '@cumulus/api/lambdas/cleanExecutions', }); -/** - * @typedef {import('@cumulus/db').PostgresExecutionRecord} PostgresExecutionRecord - * @typedef {import('knex').Knex} Knex - */ - -/** - * Extract expiration dates and identify greater and lesser bounds - * - * @param {number} payloadTimeout - Maximum number of days a record should be held onto - * @returns {Date} - */ -const getExpirationDate = ( - payloadTimeout -) => moment().subtract(payloadTimeout, 'days').toDate(); - -/** - * Clean up Elasticsearch executions that have expired - * - * @param {number} payloadTimeout - Maximum number of days a record should be held onto - * @param {boolean} cleanupRunning - Enable removal of running execution - * payloads - * @param {boolean} cleanupNonRunning - Enable removal of execution payloads for - * statuses other than 'running' - * @param {number} updateLimit - maximum number of records to update - * @param {string} index - Elasticsearch index to cleanup - * @returns {Promise} -*/ -const cleanupExpiredESExecutionPayloads = async ( - payloadTimeout, - cleanupRunning, - cleanupNonRunning, - updateLimit, - index -) => { - const _expiration = getExpirationDate(payloadTimeout); - const expiration = _expiration.getTime(); - - const must = [ - { range: { updatedAt: { lte: expiration } } }, - { - bool: { - should: [ - { exists: { field: 'finalPayload' } }, - { exists: { field: 'originalPayload' } }, - ], - }, - }, - ]; - const mustNot = []; - - if (cleanupRunning && !cleanupNonRunning) { - must.push({ term: { status: 'running' } }); - } else if (!cleanupRunning && cleanupNonRunning) { - mustNot.push({ term: { status: 'running' } }); - } - const removePayloadScript = "ctx._source.remove('finalPayload'); ctx._source.remove('originalPayload')"; - - const script = { inline: removePayloadScript }; - const body = { - query: { - bool: { - must, - mustNot, - }, - }, - script: script, - }; - const esClient = await getEsClient(); - const [{ node }] = await esConfig(); - // this launches the job for ES to perform, asynchronously - const updateTask = await esClient._client.updateByQuery({ - index, - type: 'execution', - size: updateLimit, - body, - conflicts: 'proceed', - wait_for_completion: false, - refresh: true, - }); - let taskStatus; - // this async and poll method allows us to avoid http timeouts - // and persist in case of lambda timeout - log.info(`launched async elasticsearch task id ${updateTask.body.task} - to check on this task outside this lambda, or to stop this task run the following`); - log.info(` > curl --request GET ${node}/_tasks/${updateTask.body.task}`); - log.info(` > curl --request POST ${node}/_tasks/${updateTask.body.task}/_cancel`); - do { - sleep(10000); - // eslint-disable-next-line no-await-in-loop - taskStatus = await esClient._client?.tasks.get({ task_id: updateTask.body.task }); - } while (taskStatus?.body.completed === false); - log.info(`elasticsearch task completed with status ${JSON.stringify(taskStatus?.body.task.status)}`); -}; /** * parse out environment variable configuration * @returns {{ @@ -135,33 +45,9 @@ const parseEnvironment = () => { }; }; -/** - * parse environment variables to extract configuration and run cleanup of ES executions - * - * @returns {Promise} - */ -async function cleanExecutionPayloads() { +function handler(_event) { const envConfig = parseEnvironment(); - log.info(`running cleanExecutions with configuration ${JSON.stringify(envConfig)}`); - const { - updateLimit, - cleanupRunning, - cleanupNonRunning, - payloadTimeout, - esIndex, - } = envConfig; - - await cleanupExpiredESExecutionPayloads( - payloadTimeout, - cleanupRunning, - cleanupNonRunning, - updateLimit, - esIndex - ); -} - -async function handler(_event) { - return await cleanExecutionPayloads(); + log.info(`running empty (to be updated) cleanExecutions with configuration ${JSON.stringify(envConfig)}`); } if (require.main === module) { @@ -176,7 +62,4 @@ if (require.main === module) { module.exports = { handler, - cleanExecutionPayloads, - getExpirationDate, - cleanupExpiredESExecutionPayloads, }; diff --git a/packages/api/lambdas/create-reconciliation-report-types.js b/packages/api/lambdas/create-reconciliation-report-types.js new file mode 100644 index 00000000000..eac05e81296 --- /dev/null +++ b/packages/api/lambdas/create-reconciliation-report-types.js @@ -0,0 +1,59 @@ +/** + * @typedef {import('@cumulus/types/api/files').ApiFile} ApiFile + */ + +/** + * @typedef {Object} Env + * @property {string} [CONCURRENCY] - The concurrency level for processing. + * @property {string} [AWS_REGION] - The AWS region. + * @property {string} [AWS_ACCESS_KEY_ID] - The AWS access key ID. + * @property {string} [AWS_SECRET_ACCESS_KEY] - The AWS secret access key. + * @property {string} [AWS_SESSION_TOKEN] - The AWS session token. + * @property {string} [NODE_ENV] - The Node.js environment (e.g., 'development', 'production'). + * @property {string} [DATABASE_URL] - The database connection URL. + * @property {string} [key] string - Any other environment variable as a string. + */ + +/** + * @typedef {Object} CMRCollectionItem + * @property {Object} umm - The UMM (Unified Metadata Model) object for the granule. + * @property {string} umm.ShortName - The short name of the collection. + * @property {string} umm.Version - The version of the collection. + * @property {Array} umm.RelatedUrls - The related URLs for the granule. + */ + +/** + * @typedef {Object} CMRItem + * @property {Object} umm - The UMM (Unified Metadata Model) object for the granule. + * @property {string} umm.GranuleUR - The unique identifier for the granule in CMR. + * @property {Object} umm.CollectionReference - The collection reference object. + * @property {string} umm.CollectionReference.ShortName - The short name of the collection. + * @property {string} umm.CollectionReference.Version - The version of the collection. + * @property {Array} umm.RelatedUrls - The related URLs for the granule. + */ + +/** + * @typedef {Object} FilesReport + * @property {number} okCount + * @property {ApiFile[]} onlyInCumulus + * @property {ApiFile[]} onlyInCmr + * + */ + +/** + * @typedef {Object} GranulesReport + * @property {number} okCount - The count of OK granules. + * @property {Array<{GranuleUR: string, ShortName: string, Version: string}>} onlyInCmr + * - The list of granules only in Cumulus. + * @property {Array<{granuleId: string, collectionId: string}>} onlyInCumulus + */ + +/** + * @typedef {Object} FilesInCumulus + * @property {number} okCount + * @property {Object} okCountByGranule + * @property {string[]} onlyInS3 + * @property {Object[]} onlyInDb + */ + +module.exports = {}; diff --git a/packages/api/lambdas/create-reconciliation-report.js b/packages/api/lambdas/create-reconciliation-report.js index ad940777704..0269898bd06 100644 --- a/packages/api/lambdas/create-reconciliation-report.js +++ b/packages/api/lambdas/create-reconciliation-report.js @@ -1,3 +1,5 @@ +//@ts-check + 'use strict'; const cloneDeep = require('lodash/cloneDeep'); @@ -11,32 +13,33 @@ const S3ListObjectsV2Queue = require('@cumulus/aws-client/S3ListObjectsV2Queue') const { s3 } = require('@cumulus/aws-client/services'); const BucketsConfig = require('@cumulus/common/BucketsConfig'); const { getBucketsConfigKey } = require('@cumulus/common/stack'); +const { removeNilProperties } = require('@cumulus/common/util'); const { fetchDistributionBucketMap } = require('@cumulus/distribution-utils'); const { constructCollectionId, deconstructCollectionId } = require('@cumulus/message/Collections'); const { CMRSearchConceptQueue } = require('@cumulus/cmr-client'); const { constructOnlineAccessUrl, getCmrSettings } = require('@cumulus/cmrjs/cmr-utils'); const { + CollectionSearch, getFilesAndGranuleInfoQuery, + getGranulesByApiPropertiesQuery, getKnexClient, + getUniqueCollectionsByGranuleFilter, QuerySearchClient, + translatePostgresFileToApiFile, } = require('@cumulus/db'); -const { ESCollectionGranuleQueue } = require('@cumulus/es-client/esCollectionGranuleQueue'); -const Collection = require('@cumulus/es-client/collections'); -const { ESSearchQueue } = require('@cumulus/es-client/esSearchQueue'); -const { indexReconciliationReport } = require('@cumulus/es-client/indexer'); -const { getEsClient } = require('@cumulus/es-client/search'); const Logger = require('@cumulus/logger'); -const { createInternalReconciliationReport } = require('./internal-reconciliation-report'); +const { + ReconciliationReportPgModel, + translatePostgresReconReportToApiReconReport, +} = require('@cumulus/db'); const { createGranuleInventoryReport } = require('./reports/granule-inventory-report'); const { createOrcaBackupReconciliationReport } = require('./reports/orca-backup-reconciliation-report'); -const { ReconciliationReport } = require('../models'); const { errorify, filenamify } = require('../lib/utils'); const { cmrGranuleSearchParams, - convertToESCollectionSearchParams, - convertToESGranuleSearchParams, + convertToDBGranuleSearchParams, initialReportHeader, } = require('../lib/reconciliationReport'); @@ -44,6 +47,31 @@ const log = new Logger({ sender: '@api/lambdas/create-reconciliation-report' }); const isDataBucket = (bucketConfig) => ['private', 'public', 'protected'].includes(bucketConfig.type); +// Typescript annotations + +/** + * @typedef {typeof process.env } ProcessEnv + * @typedef {import('knex').Knex} Knex + * @typedef {import('../lib/types').NormalizedRecReportParams } NormalizedRecReportParams + * @typedef {import('../lib/types').EnhancedNormalizedRecReportParams} + * EnhancedNormalizedRecReportParams + * @typedef {import('@cumulus/cmr-client/CMR').CMRConstructorParams} CMRSettings + * @typedef {import('@cumulus/db').PostgresReconciliationReportRecord} + * PostgresReconciliationReportRecord + * @typedef {import('@cumulus/types/api/reconciliation_reports').ReconciliationReportStatus} + * ReconciliationReportStatus + * @typedef {import('@cumulus/types/api/reconciliation_reports').ReconciliationReportType} + * ReconciliationReportType + * @typedef {import('@cumulus/types/api/files').ApiFile} ApiFile + * @typedef {import('@cumulus/db').PostgresGranuleRecord} PostgresGranuleRecord + * @typedef {import('./create-reconciliation-report-types').Env } Env + * @typedef {import('./create-reconciliation-report-types').CMRCollectionItem } CMRCollectionItem + * @typedef {import('./create-reconciliation-report-types').CMRItem } CMRItem + * @typedef {import('./create-reconciliation-report-types').FilesReport } FilesReport + * @typedef {import('./create-reconciliation-report-types').GranulesReport } GranulesReport + * @typedef {import('./create-reconciliation-report-types').FilesInCumulus } FilesInCumulus + */ + /** * * @param {string} reportType - reconciliation report type @@ -99,41 +127,32 @@ function isOneWayGranuleReport(reportParams) { } /** - * Checks to see if the searchParams have any value that would require a - * filtered search in ES - * @param {Object} searchParams - * @returns {boolean} returns true if searchParams contain a key that causes filtering to occur. - */ -function shouldAggregateGranulesForCollections(searchParams) { - return [ - 'updatedAt__from', - 'updatedAt__to', - 'granuleId__in', - 'provider__in', - ].some((e) => !!searchParams[e]); -} - -/** - * fetch CMR collections and filter the returned UMM CMR collections by the desired collectionIds + * Fetches collections from the CMR (Common Metadata Repository) and returns their IDs. + * + * @param {EnhancedNormalizedRecReportParams} recReportParams - The parameters for the function. + * @returns {Promise} A promise that resolves to an array of collection IDs from the CMR. * - * @param {Object} recReportParams - input report params - * @param {Array} recReportParams.collectionIds - array of collectionIds to keep - * @returns {Array} filtered list of collectionIds returned from CMR + * @example + * await fetchCMRCollections({ collectionIds: ['COLLECTION_1', 'COLLECTION_2'] }); */ async function fetchCMRCollections({ collectionIds }) { const cmrSettings = await getCmrSettings(); - const cmrCollectionsIterator = new CMRSearchConceptQueue({ - cmrSettings, - type: 'collections', - format: 'umm_json', - }); + const cmrCollectionsIterator = /** @type {CMRSearchConceptQueue} */( + new CMRSearchConceptQueue({ + cmrSettings, + type: 'collections', + format: 'umm_json', + })); const allCmrCollectionIds = []; let nextCmrItem = await cmrCollectionsIterator.shift(); while (nextCmrItem) { - allCmrCollectionIds - .push(constructCollectionId(nextCmrItem.umm.ShortName, nextCmrItem.umm.Version)); - nextCmrItem = await cmrCollectionsIterator.shift(); // eslint-disable-line no-await-in-loop + allCmrCollectionIds.push( + constructCollectionId(nextCmrItem.umm.ShortName, nextCmrItem.umm.Version) + ); + nextCmrItem + // eslint-disable-next-line no-await-in-loop + = /** @type {CMRCollectionItem | null} */ (await cmrCollectionsIterator.shift()); } const cmrCollectionIds = allCmrCollectionIds.sort(); @@ -143,31 +162,42 @@ async function fetchCMRCollections({ collectionIds }) { } /** - * Fetch collections in Elasticsearch. - * @param {Object} recReportParams - input report params. - * @returns {Promise} - list of collectionIds that match input paramaters + * Fetches collections from the database based on the provided parameters. + * + * @param {EnhancedNormalizedRecReportParams} recReportParams - The reconciliation + * report parameters. + * @returns {Promise} A promise that resolves to an array of collection IDs. */ -async function fetchESCollections(recReportParams) { - const esCollectionSearchParams = convertToESCollectionSearchParams(recReportParams); - const esGranuleSearchParams = convertToESGranuleSearchParams(recReportParams); - let esCollectionIds; - // [MHS, 09/02/2020] We are doing these two because we can't use - // aggregations on scrolls yet until we update elasticsearch version. - if (shouldAggregateGranulesForCollections(esGranuleSearchParams)) { - // Build an ESCollection and call the aggregateGranuleCollections to - // get list of collection ids that have granules that have been updated - const esCollection = new Collection({ queryStringParameters: esGranuleSearchParams }, 'collection', process.env.ES_INDEX); - const esCollectionItems = await esCollection.aggregateGranuleCollections(); - esCollectionIds = esCollectionItems.sort(); - } else { - // return all ES collections - const esCollection = new ESSearchQueue(esCollectionSearchParams, 'collection', process.env.ES_INDEX); - const esCollectionItems = await esCollection.empty(); - esCollectionIds = esCollectionItems.map( - (item) => constructCollectionId(item.name, item.version) - ).sort(); +async function fetchDbCollections(recReportParams) { + const { + collectionIds, + endTimestamp, + granuleIds, + knex, + providers, + startTimestamp, + } = recReportParams; + if (providers || granuleIds || startTimestamp || endTimestamp) { + const filteredDbCollections = await getUniqueCollectionsByGranuleFilter({ + ...recReportParams, + }); + return filteredDbCollections.map((collection) => + constructCollectionId(collection.name, collection.version)); } - return esCollectionIds; + + const queryStringParameters = removeNilProperties({ + _id__in: collectionIds ? collectionIds.join(',') : undefined, + timestamp__from: startTimestamp, + timestamp__to: endTimestamp, + sort_key: ['name', 'version'], + collate: 'C', + }); + const searchResponse = await new CollectionSearch({ + queryStringParameters: { ...queryStringParameters, limit: null }, + }).query(knex); + const dbCollections = searchResponse.results; + return dbCollections.map((collection) => + constructCollectionId(collection.name, collection.version)); } /** @@ -175,7 +205,7 @@ async function fetchESCollections(recReportParams) { * PostgreSQL, and that there are no extras in either S3 or PostgreSQL * * @param {string} Bucket - the bucket containing files to be reconciled - * @param {Object} recReportParams - input report params. + * @param {EnhancedNormalizedRecReportParams} recReportParams - input report params. * @returns {Promise} a report */ async function createReconciliationReportForBucket(Bucket, recReportParams) { @@ -279,8 +309,8 @@ async function createReconciliationReportForBucket(Bucket, recReportParams) { /** * Compare the collection holdings in CMR with Cumulus * - * @param {Object} recReportParams - lambda's input filtering parameters to - * narrow limit of report. + * @param {EnhancedNormalizedRecReportParams} recReportParams - lambda's input filtering + * parameters to narrow limit of report. * @returns {Promise} an object with the okCollections, onlyInCumulus and * onlyInCmr */ @@ -302,17 +332,20 @@ async function reconciliationReportForCollections(recReportParams) { // get all collections from CMR and sort them, since CMR query doesn't support // 'Version' as sort_key log.debug('Fetching collections from CMR.'); - const cmrCollectionIds = await fetchCMRCollections(recReportParams); - const esCollectionIds = await fetchESCollections(recReportParams); - log.info(`Comparing ${cmrCollectionIds.length} CMR collections to ${esCollectionIds.length} Elasticsearch collections`); + const cmrCollectionIds = (await fetchCMRCollections(recReportParams)).sort(); + const dbCollectionIds = (await fetchDbCollections(recReportParams)).sort(); + + log.info(`Comparing ${cmrCollectionIds.length} CMR collections to ${dbCollectionIds.length} PostgreSQL collections`); - let nextDbCollectionId = esCollectionIds[0]; + /** @type {string | undefined } */ + let nextDbCollectionId = dbCollectionIds[0]; + /** @type {string | undefined } */ let nextCmrCollectionId = cmrCollectionIds[0]; while (nextDbCollectionId && nextCmrCollectionId) { if (nextDbCollectionId < nextCmrCollectionId) { // Found an item that is only in Cumulus database and not in cmr - esCollectionIds.shift(); + dbCollectionIds.shift(); collectionsOnlyInCumulus.push(nextDbCollectionId); } else if (nextDbCollectionId > nextCmrCollectionId) { // Found an item that is only in cmr and not in Cumulus database @@ -321,16 +354,16 @@ async function reconciliationReportForCollections(recReportParams) { } else { // Found an item that is in both cmr and database okCollections.push(nextDbCollectionId); - esCollectionIds.shift(); + dbCollectionIds.shift(); cmrCollectionIds.shift(); } - nextDbCollectionId = (esCollectionIds.length !== 0) ? esCollectionIds[0] : undefined; + nextDbCollectionId = (dbCollectionIds.length !== 0) ? dbCollectionIds[0] : undefined; nextCmrCollectionId = (cmrCollectionIds.length !== 0) ? cmrCollectionIds[0] : undefined; } // Add any remaining database items to the report - collectionsOnlyInCumulus = collectionsOnlyInCumulus.concat(esCollectionIds); + collectionsOnlyInCumulus = collectionsOnlyInCumulus.concat(dbCollectionIds); // Add any remaining CMR items to the report if (!oneWayReport) collectionsOnlyInCmr = collectionsOnlyInCmr.concat(cmrCollectionIds); @@ -358,6 +391,10 @@ async function reconciliationReportForCollections(recReportParams) { * @returns {Promise} - an object with the okCount, onlyInCumulus, onlyInCmr */ async function reconciliationReportForGranuleFiles(params) { + if (!process.env.DISTRIBUTION_ENDPOINT) { + throw new Error('DISTRIBUTION_ENDPOINT is not defined in function environment variables, but is required'); + } + const distEndpoint = process.env.DISTRIBUTION_ENDPOINT; const { granuleInDb, granuleInCmr, bucketsConfig, distributionBucketMap } = params; let okCount = 0; const onlyInCumulus = []; @@ -387,7 +424,7 @@ async function reconciliationReportForGranuleFiles(params) { // not all files should be in CMR const distributionAccessUrl = await constructOnlineAccessUrl({ file: granuleFiles[urlFileName], - distEndpoint: process.env.DISTRIBUTION_ENDPOINT, + distEndpoint, bucketTypes, urlType: 'distribution', distributionBucketMap, @@ -395,7 +432,7 @@ async function reconciliationReportForGranuleFiles(params) { const s3AccessUrl = await constructOnlineAccessUrl({ file: granuleFiles[urlFileName], - distEndpoint: process.env.DISTRIBUTION_ENDPOINT, + distEndpoint, bucketTypes, urlType: 's3', distributionBucketMap, @@ -462,14 +499,17 @@ exports.reconciliationReportForGranuleFiles = reconciliationReportForGranuleFile /** * Compare the granule holdings in CMR with Cumulus for a given collection * - * @param {Object} params - parameters - * @param {string} params.collectionId - the collection which has the granules to be - * reconciled - * @param {Object} params.bucketsConfig - bucket configuration object - * @param {Object} params.distributionBucketMap - mapping of bucket->distirubtion path values - * (e.g. { bucket: distribution path }) - * @param {Object} params.recReportParams - Lambda report paramaters for narrowing focus - * @returns {Promise} - an object with the granulesReport and filesReport + * @param {Object} params - parameters + * @param {string} params.collectionId - the collection which has the granules to be + * reconciled + * @param {Object} params.bucketsConfig - bucket configuration object + * @param {Object} params.distributionBucketMap - mapping of bucket->distirubtion path values + * (e.g. { bucket: distribution path }) + * @param {EnhancedNormalizedRecReportParams} params.recReportParams - Lambda report paramaters for + * narrowing focus database + * @returns {Promise<{ granulesReport: GranulesReport, filesReport: FilesReport }>} + * - an object with the granulesReport and + * filesReport */ async function reconciliationReportForGranules(params) { // compare granule holdings: @@ -479,41 +519,56 @@ async function reconciliationReportForGranules(params) { // Report granules only in CUMULUS log.info(`reconciliationReportForGranules(${params.collectionId})`); const { collectionId, bucketsConfig, distributionBucketMap, recReportParams } = params; + const { knex } = recReportParams; const { name, version } = deconstructCollectionId(collectionId); + + /** @type {GranulesReport} */ const granulesReport = { okCount: 0, onlyInCumulus: [], onlyInCmr: [] }; + /** @type {FilesReport} */ const filesReport = { okCount: 0, onlyInCumulus: [], onlyInCmr: [] }; try { - const cmrSettings = await getCmrSettings(); - const searchParams = new URLSearchParams({ short_name: name, version: version, sort_key: ['granule_ur'] }); + const cmrSettings = /** @type CMRSettings */(await getCmrSettings()); + const searchParams = new URLSearchParams({ short_name: name, version: version, sort_key: 'granule_ur' }); cmrGranuleSearchParams(recReportParams).forEach(([paramName, paramValue]) => { searchParams.append(paramName, paramValue); }); log.debug(`fetch CMRSearchConceptQueue(${collectionId}) with searchParams: ${JSON.stringify(searchParams)}`); - const cmrGranulesIterator = new CMRSearchConceptQueue({ + const cmrGranulesIterator + = /** @type {CMRSearchConceptQueue} */(new CMRSearchConceptQueue({ cmrSettings, type: 'granules', searchParams, format: 'umm_json', + })); + + const dbSearchParams = convertToDBGranuleSearchParams({ + ...recReportParams, + collectionIds: [collectionId], + }); + const granulesSearchQuery = getGranulesByApiPropertiesQuery({ + knex, + searchParams: { ...dbSearchParams, collate: 'C' }, + sortByFields: 'granules.granule_id', }); - const esGranuleSearchParamsByCollectionId = convertToESGranuleSearchParams( - { ...recReportParams, collectionIds: [collectionId] } - ); + const pgGranulesIterator = + /** @type {QuerySearchClient} */ ( + new QuerySearchClient( + granulesSearchQuery, + 100 // arbitrary limit on how items are fetched at once + ) + ); - log.debug(`Create ES granule iterator with ${JSON.stringify(esGranuleSearchParamsByCollectionId)}`); - const esGranulesIterator = new ESCollectionGranuleQueue( - esGranuleSearchParamsByCollectionId, process.env.ES_INDEX - ); const oneWay = isOneWayGranuleReport(recReportParams); log.debug(`is oneWay granule report: ${collectionId}, ${oneWay}`); let [nextDbItem, nextCmrItem] = await Promise.all( - [esGranulesIterator.peek(), cmrGranulesIterator.peek()] + [(pgGranulesIterator.peek()), cmrGranulesIterator.peek()] ); while (nextDbItem && nextCmrItem) { - const nextDbGranuleId = nextDbItem.granuleId; + const nextDbGranuleId = nextDbItem.granule_id; const nextCmrGranuleId = nextCmrItem.umm.GranuleUR; if (nextDbGranuleId < nextCmrGranuleId) { @@ -522,7 +577,7 @@ async function reconciliationReportForGranules(params) { granuleId: nextDbGranuleId, collectionId: collectionId, }); - await esGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + await pgGranulesIterator.shift(); // eslint-disable-line no-await-in-loop } else if (nextDbGranuleId > nextCmrGranuleId) { // Found an item that is only in CMR and not in Cumulus database if (!oneWay) { @@ -536,10 +591,16 @@ async function reconciliationReportForGranules(params) { } else { // Found an item that is in both CMR and Cumulus database granulesReport.okCount += 1; + // eslint-disable-next-line no-await-in-loop + const postgresGranuleFiles = await getFilesAndGranuleInfoQuery({ + knex, + searchParams: { granule_cumulus_id: nextDbItem.cumulus_id }, + sortColumns: ['key'], + }); const granuleInDb = { granuleId: nextDbGranuleId, collectionId: collectionId, - files: nextDbItem.files, + files: postgresGranuleFiles.map((f) => translatePostgresFileToApiFile(f)), }; const granuleInCmr = { GranuleUR: nextCmrGranuleId, @@ -547,7 +608,7 @@ async function reconciliationReportForGranules(params) { Version: nextCmrItem.umm.CollectionReference.Version, RelatedUrls: nextCmrItem.umm.RelatedUrls, }; - await esGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + await pgGranulesIterator.shift(); // eslint-disable-line no-await-in-loop await cmrGranulesIterator.shift(); // eslint-disable-line no-await-in-loop // compare the files now to avoid keeping the granules' information in memory @@ -560,14 +621,17 @@ async function reconciliationReportForGranules(params) { filesReport.onlyInCmr = filesReport.onlyInCmr.concat(fileReport.onlyInCmr); } - [nextDbItem, nextCmrItem] = await Promise.all([esGranulesIterator.peek(), cmrGranulesIterator.peek()]); // eslint-disable-line max-len, no-await-in-loop + [nextDbItem, nextCmrItem] = await Promise.all([pgGranulesIterator.peek(), cmrGranulesIterator.peek()]); // eslint-disable-line max-len, no-await-in-loop } - // Add any remaining ES/PostgreSQL items to the report - while (await esGranulesIterator.peek()) { // eslint-disable-line no-await-in-loop - const dbItem = await esGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + // Add any remaining PostgreSQL items to the report + while (await pgGranulesIterator.peek()) { // eslint-disable-line no-await-in-loop + const dbItem = await pgGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + if (!dbItem) { + throw new Error('database returned item is null in reconciliationReportForGranules'); + } granulesReport.onlyInCumulus.push({ - granuleId: dbItem.granuleId, + granuleId: dbItem.granule_id, collectionId: collectionId, }); } @@ -576,6 +640,9 @@ async function reconciliationReportForGranules(params) { if (!oneWay) { while (await cmrGranulesIterator.peek()) { // eslint-disable-line no-await-in-loop const cmrItem = await cmrGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + if (!cmrItem) { + throw new Error('CMR returned item is null in reconciliationReportForGranules'); + } granulesReport.onlyInCmr.push({ GranuleUR: cmrItem.umm.GranuleUR, ShortName: nextCmrItem.umm.CollectionReference.ShortName, @@ -605,20 +672,21 @@ exports.reconciliationReportForGranules = reconciliationReportForGranules; /** * Compare the holdings in CMR with Cumulus' internal data store, report any discrepancies * - * @param {Object} params . - parameters - * @param {Object} params.bucketsConfig - bucket configuration object - * @param {Object} params.distributionBucketMap - mapping of bucket->distirubtion path values + * @param {Object} params . - parameters + * @param {Object} params.bucketsConfig - bucket configuration object + * @param {Object} params.distributionBucketMap - mapping of bucket->distirubtion path values * (e.g. { bucket: distribution path }) - * @param {Object} [params.recReportParams] - optional Lambda endpoint's input params to - * narrow report focus - * @param {number} [params.recReportParams.StartTimestamp] - * @param {number} [params.recReportParams.EndTimestamp] - * @param {string} [params.recReportparams.collectionIds] - * @returns {Promise} - a reconciliation report + * @param {EnhancedNormalizedRecReportParams} params.recReportParams - Lambda endpoint's input + * params to narrow focus of report + * @returns {Promise} - a reconciliation report */ async function reconciliationReportForCumulusCMR(params) { log.info(`reconciliationReportForCumulusCMR with params ${JSON.stringify(params)}`); - const { bucketsConfig, distributionBucketMap, recReportParams } = params; + const { + bucketsConfig, + distributionBucketMap, + recReportParams, + } = params; const collectionReport = await reconciliationReportForCollections(recReportParams); const collectionsInCumulusCmr = { okCount: collectionReport.okCollections.length, @@ -666,7 +734,7 @@ async function reconciliationReportForCumulusCMR(params) { * @param {Object} report - report to upload * @param {string} systemBucket - system bucket * @param {string} reportKey - report key - * @returns {Promise} + * @returns - A promise that resolves with the status of the return object */ function _uploadReportToS3(report, systemBucket, reportKey) { return s3().putObject({ @@ -679,17 +747,8 @@ function _uploadReportToS3(report, systemBucket, reportKey) { /** * Create a Reconciliation report and save it to S3 * - * @param {Object} recReportParams - params - * @param {Object} recReportParams.reportType - the report type - * @param {moment} recReportParams.createStartTime - when the report creation was begun - * @param {moment} recReportParams.endTimestamp - ending report datetime ISO Timestamp - * @param {string} recReportParams.location - location to inventory for report - * @param {string} recReportParams.reportKey - the s3 report key - * @param {string} recReportParams.stackName - the name of the CUMULUS stack - * @param {moment} recReportParams.startTimestamp - beginning report datetime ISO timestamp - * @param {string} recReportParams.systemBucket - the name of the CUMULUS system bucket - * @param {Knex} recReportParams.knex - Database client for interacting with PostgreSQL database - * @returns {Promise} a Promise that resolves when the report has been + * @param {EnhancedNormalizedRecReportParams} recReportParams - params + * @returns - a Promise that resolves when the report has been * uploaded to S3 */ async function createReconciliationReport(recReportParams) { @@ -698,7 +757,6 @@ async function createReconciliationReport(recReportParams) { stackName, systemBucket, location, - knex, } = recReportParams; log.info(`createReconciliationReport (${JSON.stringify(recReportParams)})`); // Fetch the bucket names to reconcile @@ -711,6 +769,7 @@ async function createReconciliationReport(recReportParams) { const bucketsConfig = new BucketsConfig(bucketsConfigJson); // Write an initial report to S3 + /** @type {FilesInCumulus} */ const filesInCumulus = { okCount: 0, okCountByGranule: {}, @@ -723,6 +782,7 @@ async function createReconciliationReport(recReportParams) { onlyInCumulus: [], onlyInCmr: [], }; + let report = { ...initialReportHeader(recReportParams), filesInCumulus, @@ -740,7 +800,7 @@ async function createReconciliationReport(recReportParams) { // Create a report for each bucket const promisedBucketReports = dataBuckets.map( - (bucket) => createReconciliationReportForBucket(bucket, recReportParams, knex) + (bucket) => createReconciliationReportForBucket(bucket, recReportParams) ); const bucketReports = await Promise.all(promisedBucketReports); @@ -748,7 +808,9 @@ async function createReconciliationReport(recReportParams) { bucketReports.forEach((bucketReport) => { report.filesInCumulus.okCount += bucketReport.okCount; - report.filesInCumulus.onlyInS3 = report.filesInCumulus.onlyInS3.concat(bucketReport.onlyInS3); // eslint-disable-line max-len + report.filesInCumulus.onlyInS3 = report.filesInCumulus.onlyInS3.concat( + bucketReport.onlyInS3 + ); report.filesInCumulus.onlyInDb = report.filesInCumulus.onlyInDb.concat( bucketReport.onlyInDb ); @@ -762,7 +824,7 @@ async function createReconciliationReport(recReportParams) { + bucketGranuleCount; }); } else { - delete report.filesInCumulus.okCountByGranule; + report.filesInCumulus.okCountByGranule = {}; } }); } @@ -801,10 +863,11 @@ async function createReconciliationReport(recReportParams) { * @param {Object} params - params * @param {string} params.systemBucket - the name of the CUMULUS system bucket * @param {string} params.stackName - the name of the CUMULUS stack - * @param {string} params.reportType - the type of reconciliation report + * @param {ReconciliationReportType} params.reportType - the type of reconciliation report * @param {string} params.reportName - the name of the report + * @param {Env} params.env - the environment variables * @param {Knex} params.knex - Optional Instance of a Knex client for testing - * @returns {Object} report record saved to the database + * @returns {Promise} report record saved to the database */ async function processRequest(params) { log.info(`processing reconciliation report request with params: ${JSON.stringify(params)}`); @@ -814,8 +877,7 @@ async function processRequest(params) { reportName, systemBucket, stackName, - esClient = await getEsClient(), - knex = await getKnexClient(env), + knex = await getKnexClient({ env }), } = params; const createStartTime = moment.utc(); const reportRecordName = reportName @@ -824,18 +886,20 @@ async function processRequest(params) { if (reportType === 'Granule Inventory') reportKey = reportKey.replace('.json', '.csv'); // add request to database - const reconciliationReportModel = new ReconciliationReport(); - const reportRecord = { + const reconciliationReportPgModel = new ReconciliationReportPgModel(); + const builtReportRecord = { name: reportRecordName, type: reportType, + /** @type ReconciliationReportStatus */ status: 'Pending', location: buildS3Uri(systemBucket, reportKey), }; - let apiRecord = await reconciliationReportModel.create(reportRecord); - await indexReconciliationReport(esClient, apiRecord, process.env.ES_INDEX); - log.info(`Report added to database as pending: ${JSON.stringify(apiRecord)}.`); + let [reportPgRecord] = await reconciliationReportPgModel.create(knex, builtReportRecord); + // api format was being logged prior to ES removal, so keeping format for consistency + let reportApiRecord = translatePostgresReconReportToApiReconReport(reportPgRecord); + log.info(`Report added to database as Pending: ${JSON.stringify(reportApiRecord)}.`); - const concurrency = env.CONCURRENCY || 3; + const concurrency = env.CONCURRENCY || '3'; try { const recReportParams = { @@ -848,46 +912,56 @@ async function processRequest(params) { }; log.info(`Beginning ${reportType} report with params: ${JSON.stringify(recReportParams)}`); if (reportType === 'Internal') { - await createInternalReconciliationReport(recReportParams); + log.error( + 'Internal Reconciliation Reports are no longer valid, as Cumulus is no longer utilizing Elasticsearch' + ); + throw new Error('Internal Reconciliation Reports are no longer valid'); } else if (reportType === 'Granule Inventory') { await createGranuleInventoryReport(recReportParams); } else if (reportType === 'ORCA Backup') { await createOrcaBackupReconciliationReport(recReportParams); - } else { + } else if (['Inventory', 'Granule Not Found'].includes(reportType)) { // reportType is in ['Inventory', 'Granule Not Found'] await createReconciliationReport(recReportParams); } - apiRecord = await reconciliationReportModel.updateStatus({ name: reportRecord.name }, 'Generated'); - await indexReconciliationReport(esClient, { ...apiRecord, status: 'Generated' }, process.env.ES_INDEX); + + const generatedRecord = { + ...reportPgRecord, + /** @type ReconciliationReportStatus */ + status: 'Generated', + }; + [reportPgRecord] = await reconciliationReportPgModel.upsert(knex, generatedRecord); } catch (error) { - log.error(`Error caught in createReconciliationReport creating ${reportType} report ${reportRecordName}. ${error}`); - const updates = { + log.error(`Error caught in createReconciliationReport creating ${reportType} report ${reportRecordName}. ${error}`); // eslint-disable-line max-len + const erroredRecord = { + ...reportPgRecord, + /** @type ReconciliationReportStatus */ status: 'Failed', error: { Error: error.message, Cause: errorify(error), }, }; - apiRecord = await reconciliationReportModel.update({ name: reportRecord.name }, updates); - await indexReconciliationReport( - esClient, - { ...apiRecord, ...updates }, - process.env.ES_INDEX - ); + [reportPgRecord] = await reconciliationReportPgModel.upsert(knex, erroredRecord); + reportApiRecord = translatePostgresReconReportToApiReconReport(reportPgRecord); + log.error(`Report updated in database as Failed including error: ${JSON.stringify(reportApiRecord)}`); throw error; } - return reconciliationReportModel.get({ name: reportRecord.name }); + reportPgRecord = await reconciliationReportPgModel.get(knex, { name: builtReportRecord.name }); + reportApiRecord = translatePostgresReconReportToApiReconReport(reportPgRecord); + log.info(`Report updated in database as Generated: ${JSON.stringify(reportApiRecord)}.`); + return reportApiRecord; } async function handler(event) { // increase the limit of search result from CMR.searchCollections/searchGranules - process.env.CMR_LIMIT = process.env.CMR_LIMIT || 5000; - process.env.CMR_PAGE_SIZE = process.env.CMR_PAGE_SIZE || 200; + process.env.CMR_LIMIT = process.env.CMR_LIMIT || '5000'; + process.env.CMR_PAGE_SIZE = process.env.CMR_PAGE_SIZE || '200'; - const varsToLog = ['CMR_LIMIT', 'CMR_PAGE_SIZE', 'ES_SCROLL', 'ES_SCROLL_SIZE']; + const varsToLog = ['CMR_LIMIT', 'CMR_PAGE_SIZE']; const envsToLog = pickBy(process.env, (value, key) => varsToLog.includes(key)); - log.info(`CMR and ES Environment variables: ${JSON.stringify(envsToLog)}`); + log.info(`CMR Environment variables: ${JSON.stringify(envsToLog)}`); return await processRequest(event); } diff --git a/packages/api/lambdas/index-from-database.js b/packages/api/lambdas/index-from-database.js deleted file mode 100644 index e0666992a18..00000000000 --- a/packages/api/lambdas/index-from-database.js +++ /dev/null @@ -1,324 +0,0 @@ -'use strict'; - -const isNil = require('lodash/isNil'); -const pLimit = require('p-limit'); - -const DynamoDbSearchQueue = require('@cumulus/aws-client/DynamoDbSearchQueue'); -const log = require('@cumulus/common/log'); - -const { getEsClient } = require('@cumulus/es-client/search'); -const { - CollectionPgModel, - ExecutionPgModel, - AsyncOperationPgModel, - GranulePgModel, - ProviderPgModel, - RulePgModel, - PdrPgModel, - getKnexClient, - translatePostgresCollectionToApiCollection, - translatePostgresExecutionToApiExecution, - translatePostgresAsyncOperationToApiAsyncOperation, - translatePostgresGranuleToApiGranule, - translatePostgresProviderToApiProvider, - translatePostgresPdrToApiPdr, - translatePostgresRuleToApiRule, -} = require('@cumulus/db'); -const indexer = require('@cumulus/es-client/indexer'); - -/** - * Return specified concurrency for ES requests. - * - * Returned value is used with [p-limit](https://github.com/sindresorhus/p-limit), which - * does not accept 0. - * - * @param {Object} event - Incoming Lambda event - * @returns {number} - Specified request concurrency. Defaults to 10. - * @throws {TypeError} - */ -const getEsRequestConcurrency = (event) => { - if (!isNil(event.esRequestConcurrency)) { - const parsedValue = Number.parseInt(event.esRequestConcurrency, 10); - - if (Number.isInteger(parsedValue) && parsedValue > 0) { - return parsedValue; - } - - throw new TypeError('event.esRequestConcurrency must be an integer greater than 0'); - } - - if (!isNil(process.env.ES_CONCURRENCY)) { - const parsedValue = Number.parseInt(process.env.ES_CONCURRENCY, 10); - - if (Number.isInteger(parsedValue) && parsedValue > 0) { - return parsedValue; - } - - throw new TypeError('The ES_CONCURRENCY environment variable must be an integer greater than 0'); - } - - return 10; -}; - -// Legacy method used for indexing Reconciliation Reports only -async function indexReconciliationReports({ - esClient, - tableName, - esIndex, - indexFn, - limitEsRequests, -}) { - const scanQueue = new DynamoDbSearchQueue({ - TableName: tableName, - }); - - let itemsComplete = false; - let totalItemsIndexed = 0; - - /* eslint-disable no-await-in-loop */ - while (itemsComplete === false) { - await scanQueue.fetchItems(); - - itemsComplete = scanQueue.items[scanQueue.items.length - 1] === null; - - if (itemsComplete) { - // pop the null item off - scanQueue.items.pop(); - } - - if (scanQueue.items.length === 0) { - log.info(`No records to index for ${tableName}`); - return true; - } - - log.info(`Attempting to index ${scanQueue.items.length} records from ${tableName}`); - - const input = scanQueue.items.map( - (item) => limitEsRequests( - async () => { - try { - return await indexFn(esClient, item, esIndex); - } catch (error) { - log.error(`Error indexing record ${JSON.stringify(item)}, error: ${error}`); - return false; - } - } - ) - ); - const results = await Promise.all(input); - const successfulResults = results.filter((result) => result !== false); - totalItemsIndexed += successfulResults; - - log.info(`Completed index of ${successfulResults.length} records from ${tableName}`); - } - /* eslint-enable no-await-in-loop */ - - return totalItemsIndexed; -} - -/** -* indexModel - Index a postgres RDS table's contents to ElasticSearch -* -* @param {Object} params -- parameters -* @param {any} params.esClient -- ElasticSearch client -* @param {any} params.postgresModel -- @cumulus/db model -* @param {string} params.esIndex -- esIndex to write records to -* @param {any} params.indexFn -- Indexer function that maps to the database model -* @param {any} params.limitEsRequests -- limitEsRequests method (used for testing) -* @param {Knex} params.knex -- configured knex instance -* @param {any} params.translationFunction -- function to translate postgres record -* to API record for ES -* @param {number} params.pageSize -- Page size for postgres pagination -* @returns {number} -- number of items indexed -*/ -async function indexModel({ - esClient, - postgresModel, - esIndex, - indexFn, - limitEsRequests, - knex, - translationFunction, - pageSize, -}) { - let startId = 1; - let totalItemsIndexed = 0; - let done; - let maxIndex = await postgresModel.getMaxCumulusId(knex); - let failCount = 0; - - log.info(`Starting index of ${postgresModel.tableName} with max cumulus_id of ${maxIndex}`); - /* eslint-disable no-await-in-loop */ - while (done !== true && maxIndex > 0) { - const pageResults = await postgresModel.paginateByCumulusId(knex, startId, pageSize); - log.info( - `Attempting to index ${pageResults.length} records from ${postgresModel.tableName}` - ); - - const indexPromises = pageResults.map((pageResult) => limitEsRequests(async () => { - let translationResult; - try { - translationResult = await translationFunction(pageResult); - await esClient.refreshClient(); - return await indexFn(esClient, translationResult, esIndex); - } catch (error) { - log.error( - `Error indexing record ${JSON.stringify(translationResult)}, error: ${error.message}` - ); - return false; - } - })); - - const results = await Promise.all(indexPromises); - const successfulResults = results.filter((result) => result !== false); - failCount += (results.length - successfulResults.length); - - totalItemsIndexed += successfulResults.length; - - log.info(`Completed index of ${successfulResults.length} records from ${postgresModel.tableName}`); - startId += pageSize; - if (startId > maxIndex) { - startId = maxIndex; - log.info(`Continuing indexing from cumulus_id ${startId} to account for new rows from ${postgresModel.tableName}`); - const oldMaxIndex = maxIndex; - maxIndex = await postgresModel.getMaxCumulusId(knex); - if (maxIndex <= oldMaxIndex) { - done = true; - } - } - } - /* eslint-enable no-await-in-loop */ - log.info(`Completed successful index of ${totalItemsIndexed} records from ${postgresModel.tableName}`); - if (failCount) { - log.warn(`${failCount} records failed indexing from ${postgresModel.tableName}`); - } - return totalItemsIndexed; -} - -async function indexFromDatabase(event) { - const { - indexName: esIndex, - esHost = process.env.ES_HOST, - reconciliationReportsTable = process.env.ReconciliationReportsTable, - postgresResultPageSize, - postgresConnectionPoolSize, - } = event; - const esClient = await getEsClient(esHost); - const knex = event.knex || (await getKnexClient({ - env: { - dbMaxPool: Number.parseInt(postgresConnectionPoolSize, 10) || 10, - ...process.env, - }, - })); - - const pageSize = Number.parseInt(postgresResultPageSize, 10) || 1000; - const esRequestConcurrency = getEsRequestConcurrency(event); - log.info( - `Tuning configuration: esRequestConcurrency: ${esRequestConcurrency}, postgresResultPageSize: ${pageSize}, postgresConnectionPoolSize: ${postgresConnectionPoolSize}` - ); - - const limitEsRequests = pLimit(esRequestConcurrency); - - await Promise.all([ - indexModel({ - esClient, - esIndex, - indexFn: indexer.indexCollection, - limitEsRequests, - postgresModel: new CollectionPgModel(), - translationFunction: translatePostgresCollectionToApiCollection, - knex, - pageSize, - }), - indexModel({ - esClient, - esIndex, - indexFn: indexer.indexExecution, - limitEsRequests, - postgresModel: new ExecutionPgModel(), - translationFunction: (record) => - translatePostgresExecutionToApiExecution(record, knex), - knex, - pageSize, - }), - indexModel({ - esClient, - esIndex, - indexFn: indexer.indexAsyncOperation, - limitEsRequests, - postgresModel: new AsyncOperationPgModel(), - translationFunction: translatePostgresAsyncOperationToApiAsyncOperation, - knex, - pageSize, - }), - indexModel({ - esClient, - esIndex, - indexFn: indexer.indexGranule, - limitEsRequests, - postgresModel: new GranulePgModel(), - translationFunction: (record) => - translatePostgresGranuleToApiGranule({ - granulePgRecord: record, - knexOrTransaction: knex, - }), - knex, - pageSize, - }), - indexModel({ - esClient, - esIndex, - indexFn: indexer.indexPdr, - limitEsRequests, - postgresModel: new PdrPgModel(), - translationFunction: (record) => - translatePostgresPdrToApiPdr(record, knex), - knex, - pageSize, - }), - indexModel({ - esClient, - esIndex, - indexFn: indexer.indexProvider, - limitEsRequests, - postgresModel: new ProviderPgModel(), - translationFunction: translatePostgresProviderToApiProvider, - knex, - pageSize, - }), - indexReconciliationReports({ - esClient, - tableName: reconciliationReportsTable, - esIndex, - indexFn: indexer.indexReconciliationReport, - limitEsRequests, - }), - indexModel({ - esClient, - esIndex, - indexFn: indexer.indexRule, - limitEsRequests, - postgresModel: new RulePgModel(), - translationFunction: (record) => - translatePostgresRuleToApiRule(record, knex), - knex, - pageSize, - }), - ]); -} - -async function handler(event) { - log.info(`Starting index from database for index ${event.indexName}`); - - await indexFromDatabase(event); - - log.info('Index from database complete'); - - return 'Index from database complete'; -} - -module.exports = { - handler, - indexFromDatabase, - getEsRequestConcurrency, -}; diff --git a/packages/api/lambdas/internal-reconciliation-report.js b/packages/api/lambdas/internal-reconciliation-report.js deleted file mode 100644 index 657da6500d6..00000000000 --- a/packages/api/lambdas/internal-reconciliation-report.js +++ /dev/null @@ -1,461 +0,0 @@ -'use strict'; - -const chunk = require('lodash/chunk'); -const cloneDeep = require('lodash/cloneDeep'); -const pick = require('lodash/pick'); -const sortBy = require('lodash/sortBy'); -const isEqual = require('lodash/isEqual'); -const intersection = require('lodash/intersection'); -const union = require('lodash/union'); -const omit = require('lodash/omit'); -const moment = require('moment'); -const pMap = require('p-map'); - -const Logger = require('@cumulus/logger'); -const { constructCollectionId } = require('@cumulus/message/Collections'); -const { s3 } = require('@cumulus/aws-client/services'); -const { ESSearchQueue } = require('@cumulus/es-client/esSearchQueue'); -const { - CollectionPgModel, - translatePostgresCollectionToApiCollection, - getKnexClient, - getCollectionsByGranuleIds, - getGranulesByApiPropertiesQuery, - QuerySearchClient, - translatePostgresGranuleResultToApiGranule, -} = require('@cumulus/db'); - -const { - convertToDBCollectionSearchObject, - convertToESCollectionSearchParams, - convertToESGranuleSearchParams, - convertToDBGranuleSearchParams, - filterDBCollections, - initialReportHeader, - compareEsGranuleAndApiGranule, -} = require('../lib/reconciliationReport'); - -const log = new Logger({ sender: '@api/lambdas/internal-reconciliation-report' }); - -/** - * Compare the collection holdings in Elasticsearch with Database - * - * @param {Object} recReportParams - lambda's input filtering parameters to - * narrow limit of report. - * @returns {Promise} an object with the okCount, onlyInEs, onlyInDb - * and withConfilcts - */ -async function internalRecReportForCollections(recReportParams) { - log.info(`internalRecReportForCollections (${JSON.stringify(recReportParams)})`); - // compare collection holdings: - // Get collection list in ES ordered by granuleId - // Get collection list in PostgreSQL ordered by granuleId - // Report collections only in ES - // Report collections only in PostgreSQL - // Report collections with different contents - - const searchParams = convertToESCollectionSearchParams(recReportParams); - const esCollectionsIterator = new ESSearchQueue( - { ...searchParams, sort_key: ['name', 'version'] }, 'collection', process.env.ES_INDEX - ); - - const collectionPgModel = new CollectionPgModel(); - const knex = recReportParams.knex || await getKnexClient(); - - // get collections from database and sort them, since the scan result is not ordered - const [ - updatedAtRangeParams, - dbSearchParams, - ] = convertToDBCollectionSearchObject(recReportParams); - - const dbCollectionsSearched = await collectionPgModel.searchWithUpdatedAtRange( - knex, - dbSearchParams, - updatedAtRangeParams - ); - - // TODO - improve this sort - const dbCollectionItems = sortBy( - filterDBCollections(dbCollectionsSearched, recReportParams), - ['name', 'version'] - ); - - let okCount = 0; - const withConflicts = []; - let onlyInEs = []; - let onlyInDb = []; - - const fieldsIgnored = ['timestamp', 'updatedAt', 'createdAt']; - let nextEsItem = await esCollectionsIterator.peek(); - let nextDbItem = dbCollectionItems.length !== 0 - ? translatePostgresCollectionToApiCollection(dbCollectionItems[0]) - : undefined; - - while (nextEsItem && nextDbItem) { - const esCollectionId = constructCollectionId(nextEsItem.name, nextEsItem.version); - const dbCollectionId = constructCollectionId(nextDbItem.name, nextDbItem.version); - - if (esCollectionId < dbCollectionId) { - // Found an item that is only in ES and not in DB - onlyInEs.push(esCollectionId); - await esCollectionsIterator.shift(); // eslint-disable-line no-await-in-loop - } else if (esCollectionId > dbCollectionId) { - // Found an item that is only in DB and not in ES - onlyInDb.push(dbCollectionId); - dbCollectionItems.shift(); - } else { - // Found an item that is in both ES and DB - if ( - isEqual( - omit(nextEsItem, fieldsIgnored), - omit( - nextDbItem, - fieldsIgnored - ) - ) - ) { - okCount += 1; - } else { - withConflicts.push({ es: nextEsItem, db: nextDbItem }); - } - await esCollectionsIterator.shift(); // eslint-disable-line no-await-in-loop - dbCollectionItems.shift(); - } - - nextEsItem = await esCollectionsIterator.peek(); // eslint-disable-line no-await-in-loop - nextDbItem = dbCollectionItems.length !== 0 - ? translatePostgresCollectionToApiCollection(dbCollectionItems[0]) - : undefined; - } - - // Add any remaining ES items to the report - onlyInEs = onlyInEs.concat( - (await esCollectionsIterator.empty()) - .map((item) => constructCollectionId(item.name, item.version)) - ); - - // Add any remaining DB items to the report - onlyInDb = onlyInDb - .concat(dbCollectionItems.map((item) => constructCollectionId(item.name, item.version))); - - return { okCount, withConflicts, onlyInEs, onlyInDb }; -} - -/** - * Get all collectionIds from ES and database combined - * - * @returns {Promise>} list of collectionIds - */ -async function getAllCollections() { - const collectionPgModel = new CollectionPgModel(); - const knex = await getKnexClient(); - - const dbCollections = (await collectionPgModel.search(knex, {})) - .map((collection) => constructCollectionId(collection.name, collection.version)); - - const esCollectionsIterator = new ESSearchQueue( - { sort_key: ['name', 'version'], fields: ['name', 'version'] }, 'collection', process.env.ES_INDEX - ); - const esCollections = (await esCollectionsIterator.empty()) - .map((item) => constructCollectionId(item.name, item.version)); - - return union(dbCollections, esCollections); -} - -async function getAllCollectionIdsByGranuleIds({ - granuleIds, - knex, - concurrency, -}) { - const collectionIds = new Set(); - await pMap( - chunk(granuleIds, 100), - async (granuleIdsBatch) => { - const collections = await getCollectionsByGranuleIds(knex, granuleIdsBatch); - collections.forEach( - (collection) => { - const collectionId = constructCollectionId(collection.name, collection.version); - collectionIds.add(collectionId); - } - ); - }, - { - concurrency, - } - ); - return [...collectionIds]; -} - -/** - * Get list of collections for the given granuleIds - * - * @param {Object} recReportParams - * @param {Array} recReportParams.granuleIds - list of granuleIds - * @returns {Promise>} list of collectionIds - */ -async function getCollectionsForGranules(recReportParams) { - const { - granuleIds, - } = recReportParams; - let dbCollectionIds = []; - log.info('Getting collection IDs by Granule IDs'); - dbCollectionIds = await getAllCollectionIdsByGranuleIds(recReportParams); - - log.info('Completed getting collection IDs'); - - const esGranulesIterator = new ESSearchQueue( - { granuleId__in: granuleIds.join(','), sort_key: ['collectionId'], fields: ['collectionId'] }, 'granule', process.env.ES_INDEX - ); - const esCollections = (await esGranulesIterator.empty()) - .map((granule) => (granule ? granule.collectionId : undefined)); - - return union(dbCollectionIds, esCollections); -} - -/** - * Get list of collections for granule search based on input filtering parameters - * - * @param {Object} recReportParams - lambda's input filtering parameters - * @returns {Promise>} list of collectionIds - */ -async function getCollectionsForGranuleSearch(recReportParams) { - const { collectionIds, granuleIds } = recReportParams; - let collections = []; - if (granuleIds) { - const collectionIdsForGranules = await getCollectionsForGranules(recReportParams); - collections = (collectionIds) - ? intersection(collectionIds, collectionIdsForGranules) - : collectionIdsForGranules; - } else { - collections = collectionIds || await getAllCollections(); - } - return collections; -} - -/** - * Compare the granule holdings for a given collection - * - * @param {string} collectionId - collection id - * @param {Object} recReportParams - lambda's input filtering parameters - * @returns {Promise} an object with the okCount, onlyInEs, onlyInDb - * and withConfilcts - */ -async function reportForGranulesByCollectionId(collectionId, recReportParams) { - // For each collection, - // Get granule list in ES ordered by granuleId - // Get granule list in PostgreSQL ordered by granuleId - // Report granules only in ES - // Report granules only in PostgreSQL - // Report granules with different contents - - const esSearchParams = convertToESGranuleSearchParams(recReportParams); - const esGranulesIterator = new ESSearchQueue( - { - ...esSearchParams, - collectionId, - sort_key: ['granuleId'], - }, - 'granule', - process.env.ES_INDEX - ); - - const searchParams = convertToDBGranuleSearchParams({ - ...recReportParams, - collectionIds: collectionId, - }); - const granulesSearchQuery = getGranulesByApiPropertiesQuery( - recReportParams.knex, - searchParams, - ['collectionName', 'collectionVersion', 'granule_id'] - ); - const pgGranulesSearchClient = new QuerySearchClient( - granulesSearchQuery, - 100 // arbitrary limit on how items are fetched at once - ); - - let okCount = 0; - const withConflicts = []; - const onlyInEs = []; - const onlyInDb = []; - const granuleFields = ['granuleId', 'collectionId', 'provider', 'createdAt', 'updatedAt']; - - let [nextEsItem, nextDbItem] = await Promise.all([esGranulesIterator.peek(), pgGranulesSearchClient.peek()]); // eslint-disable-line max-len - - /* eslint-disable no-await-in-loop */ - while (nextEsItem && nextDbItem) { - if (nextEsItem.granuleId < nextDbItem.granule_id) { - // Found an item that is only in ES and not in DB - onlyInEs.push(pick(nextEsItem, granuleFields)); - await esGranulesIterator.shift(); - } else if (nextEsItem.granuleId > nextDbItem.granule_id) { - const apiGranule = await translatePostgresGranuleResultToApiGranule( - recReportParams.knex, - nextDbItem - ); - - // Found an item that is only in DB and not in ES - onlyInDb.push(pick(apiGranule, granuleFields)); - await pgGranulesSearchClient.shift(); - } else { - const apiGranule = await translatePostgresGranuleResultToApiGranule( - recReportParams.knex, - nextDbItem - ); - - // Found an item that is in both ES and DB - if (compareEsGranuleAndApiGranule(nextEsItem, apiGranule)) { - okCount += 1; - } else { - withConflicts.push({ es: nextEsItem, db: apiGranule }); - } - await Promise.all([esGranulesIterator.shift(), pgGranulesSearchClient.shift()]); - } - - [nextEsItem, nextDbItem] = await Promise.all([esGranulesIterator.peek(), pgGranulesSearchClient.peek()]); // eslint-disable-line max-len - } - - // Add any remaining ES items to the report - while (await esGranulesIterator.peek()) { - const item = await esGranulesIterator.shift(); - onlyInEs.push(pick(item, granuleFields)); - } - - // Add any remaining DB items to the report - while (await pgGranulesSearchClient.peek()) { - const item = await pgGranulesSearchClient.shift(); - const apiGranule = await translatePostgresGranuleResultToApiGranule(recReportParams.knex, item); - onlyInDb.push(pick(apiGranule, granuleFields)); - } - /* eslint-enable no-await-in-loop */ - - return { okCount, withConflicts, onlyInEs, onlyInDb }; -} - -/** - * Compare the granule holdings in Elasticsearch with Database - * - * @param {Object} recReportParams - lambda's input filtering parameters to - * narrow limit of report. - * @returns {Promise} an object with the okCount, onlyInEs, onlyInDb - * and withConfilcts - */ -async function internalRecReportForGranules(recReportParams) { - log.debug('internal-reconciliation-report internalRecReportForGranules'); - log.info(`internalRecReportForGranules (${JSON.stringify(recReportParams)})`); - // To avoid 'scan' granules table, we query a Global Secondary Index(GSI) in granules - // table with collectionId. - // compare granule holdings: - // Get collections list from db and es based on request parameters or use the collectionId - // from the request - // For each collection, - // compare granule holdings and get report - // Report granules only in ES - // Report granules only in PostgreSQL - // Report granules with different contents - - const collections = await getCollectionsForGranuleSearch(recReportParams); - - const searchParams = omit(recReportParams, ['collectionIds']); - - const reports = await pMap( - collections, - (collectionId) => reportForGranulesByCollectionId(collectionId, searchParams), - { - concurrency: recReportParams.concurrency, - } - ); - - const report = {}; - report.okCount = reports - .reduce((accumulator, currentValue) => accumulator + currentValue.okCount, 0); - report.withConflicts = reports - .reduce((accumulator, currentValue) => accumulator.concat(currentValue.withConflicts), []); - report.onlyInEs = reports - .reduce((accumulator, currentValue) => accumulator.concat(currentValue.onlyInEs), []); - report.onlyInDb = reports - .reduce((accumulator, currentValue) => accumulator.concat(currentValue.onlyInDb), []); - - return report; -} - -/** - * Create a Internal Reconciliation report and save it to S3 - * - * @param {Object} recReportParams - params - * @param {Object} recReportParams.collectionIds - array of collectionIds - * @param {Object} recReportParams.reportType - the report type - * @param {moment} recReportParams.createStartTime - when the report creation was begun - * @param {moment} recReportParams.endTimestamp - ending report datetime ISO Timestamp - * @param {string} recReportParams.reportKey - the s3 report key - * @param {string} recReportParams.stackName - the name of the CUMULUS stack - * @param {moment} recReportParams.startTimestamp - beginning report datetime ISO timestamp - * @param {string} recReportParams.systemBucket - the name of the CUMULUS system bucket - * @returns {Promise} a Promise that resolves when the report has been - * uploaded to S3 - */ -async function createInternalReconciliationReport(recReportParams) { - log.info(`createInternalReconciliationReport parameters ${JSON.stringify(recReportParams)}`); - const { - reportKey, - systemBucket, - } = recReportParams; - - // Write an initial report to S3 - const initialReportFormat = { - okCount: 0, - withConflicts: [], - onlyInEs: [], - onlyInDb: [], - }; - - let report = { - ...initialReportHeader(recReportParams), - collections: cloneDeep(initialReportFormat), - granules: cloneDeep(initialReportFormat), - }; - - try { - await s3().putObject({ - Bucket: systemBucket, - Key: reportKey, - Body: JSON.stringify(report, undefined, 2), - }); - - const [collectionsReport, granulesReport] = await Promise.all([ - internalRecReportForCollections(recReportParams), - internalRecReportForGranules(recReportParams), - ]); - report = Object.assign(report, { collections: collectionsReport, granules: granulesReport }); - - // Create the full report - report.createEndTime = moment.utc().toISOString(); - report.status = 'SUCCESS'; - - // Write the full report to S3 - return s3().putObject({ - Bucket: systemBucket, - Key: reportKey, - Body: JSON.stringify(report, undefined, 2), - }); - } catch (error) { - log.error(`Error caught in createInternalReconciliationReport. ${error}`); - // Create the full report - report.createEndTime = moment.utc().toISOString(); - report.status = 'Failed'; - - // Write the full report to S3 - await s3().putObject({ - Bucket: systemBucket, - Key: reportKey, - Body: JSON.stringify(report, undefined, 2), - }); - throw error; - } -} - -module.exports = { - compareEsGranuleAndApiGranule, - internalRecReportForCollections, - internalRecReportForGranules, - createInternalReconciliationReport, -}; diff --git a/packages/api/lambdas/process-s3-dead-letter-archive.js b/packages/api/lambdas/process-s3-dead-letter-archive.js index ea0dc3d5541..fedec5b4cf9 100644 --- a/packages/api/lambdas/process-s3-dead-letter-archive.js +++ b/packages/api/lambdas/process-s3-dead-letter-archive.js @@ -4,9 +4,6 @@ const pSettle = require('p-settle'); const log = require('@cumulus/common/log'); -const { - getEsClient, -} = require('@cumulus/es-client/search'); const S3 = require('@cumulus/aws-client/S3'); const { s3 } = require('@cumulus/aws-client/services'); const { getJsonS3Object, deleteS3Object } = require('@cumulus/aws-client/S3'); @@ -101,13 +98,10 @@ async function processDeadLetterArchive({ let continuationToken; let allSuccessKeys = []; const allFailedKeys = []; - const esClient = await getEsClient(); let batchNumber = 1; /* eslint-disable no-await-in-loop */ do { log.info(`Processing batch ${batchNumber}`); - // Refresh ES client to avoid credentials timeout for long running processes - esClient.refreshClient(); listObjectsResponse = await s3().listObjectsV2({ Bucket: bucket, Prefix: path, @@ -120,7 +114,7 @@ async function processDeadLetterArchive({ const deadLetterMessage = await getJsonS3Object(bucket, deadLetterObject.Key); const cumulusMessage = await unwrapDeadLetterCumulusMessage(deadLetterMessage); try { - await writeRecordsFunction({ cumulusMessage, knex, esClient }); + await writeRecordsFunction({ cumulusMessage, knex }); return deadLetterObject.Key; } catch (error) { log.error(`Failed to write records from cumulusMessage for dead letter ${deadLetterObject.Key} due to '${error}'`); diff --git a/packages/api/lambdas/reports/granule-inventory-report.js b/packages/api/lambdas/reports/granule-inventory-report.js index e4f9d92e4bb..98dd7bfd7fc 100644 --- a/packages/api/lambdas/reports/granule-inventory-report.js +++ b/packages/api/lambdas/reports/granule-inventory-report.js @@ -1,3 +1,5 @@ +//@ts-check + 'use strict'; const noop = require('lodash/noop'); @@ -15,11 +17,14 @@ const log = new Logger({ sender: '@api/lambdas/granule-inventory-report' }); const { convertToDBGranuleSearchParams } = require('../../lib/reconciliationReport'); +/** + * @typedef {import('../../lib/types').EnhancedNormalizedRecReportParams} + * EnhancedNormalizedRecReportParams + */ + /** * Builds a CSV file of all granules in the Cumulus DB - * @param {Object} recReportParams - * @param {string} recReportParams.reportKey - s3 key to store report - * @param {string} recReportParams.systemBucket - bucket to store report. + * @param {EnhancedNormalizedRecReportParams} recReportParams * @returns {Promise} - promise of a report written to s3. */ async function createGranuleInventoryReport(recReportParams) { @@ -42,11 +47,11 @@ async function createGranuleInventoryReport(recReportParams) { const { reportKey, systemBucket } = recReportParams; const searchParams = convertToDBGranuleSearchParams(recReportParams); - const granulesSearchQuery = getGranulesByApiPropertiesQuery( - recReportParams.knex, + const granulesSearchQuery = getGranulesByApiPropertiesQuery({ + knex: recReportParams.knex, searchParams, - ['collectionName', 'collectionVersion', 'granule_id'] - ); + sortByFields: ['collectionName', 'collectionVersion', 'granule_id'], + }); const pgGranulesSearchClient = new QuerySearchClient( granulesSearchQuery, 100 // arbitrary limit on how items are fetched at once diff --git a/packages/api/lambdas/reports/orca-backup-reconciliation-report.js b/packages/api/lambdas/reports/orca-backup-reconciliation-report.js index bac82da2dbb..be8e50fb43f 100644 --- a/packages/api/lambdas/reports/orca-backup-reconciliation-report.js +++ b/packages/api/lambdas/reports/orca-backup-reconciliation-report.js @@ -1,3 +1,5 @@ +//@ts-check + 'use strict'; const cloneDeep = require('lodash/cloneDeep'); @@ -8,19 +10,88 @@ const set = require('lodash/set'); const moment = require('moment'); const path = require('path'); +const { + getGranulesByApiPropertiesQuery, + QuerySearchClient, + getKnexClient, + FilePgModel, +} = require('@cumulus/db'); const { s3 } = require('@cumulus/aws-client/services'); -const { ESSearchQueue } = require('@cumulus/es-client/esSearchQueue'); const Logger = require('@cumulus/logger'); -const { constructCollectionId } = require('@cumulus/message/Collections'); +const { deconstructCollectionId, constructCollectionId } = require('@cumulus/message/Collections'); +const filePgModel = new FilePgModel(); const { - convertToESCollectionSearchParams, - convertToESGranuleSearchParamsWithCreatedAtRange, + convertToDBGranuleSearchParams, convertToOrcaGranuleSearchParams, initialReportHeader, } = require('../../lib/reconciliationReport'); const ORCASearchCatalogQueue = require('../../lib/ORCASearchCatalogQueue'); +// Typedefs +/** + * @typedef {Object} ConflictFile + * @property {string} fileName + * @property {string} bucket + * @property {string} key + * @property {string} [orcaBucket] + * @property {string} reason + */ + +/** + * @typedef { import('@cumulus/db').PostgresGranuleRecord } PostgresGranuleRecord + * @typedef {import('../../lib/types').EnhancedNormalizedRecReportParams } + * EnhancedNormalizedRecReportParams + */ + +/** + * @typedef {Object} GranuleReport + * @property {boolean} ok + * @property {number} okFilesCount + * @property {number} cumulusFilesCount + * @property {number} orcaFilesCount + * @property {string} granuleId + * @property {string} collectionId + * @property {string} provider + * @property {number} createdAt + * @property {number} updatedAt + * @property {ConflictFile[]} conflictFiles + */ +/** + * @typedef {Object} CollectionConfig + */ + +/** @typedef {import('@cumulus/db').PostgresFileRecord} PostgresFileRecord */ + +/** + * @typedef {Object} OrcaReportGranuleObject + * @property {string} collectionId - The ID of the collection + * @property {string} collectionName - The name of the collection associated with the granule + * @property {string} collectionVersion - The version of + * the collection associated with the granule + * @property {string} providerName - The name of the provider associated with the granule + * @property {PostgresFileRecord[]} files - The files associated with the granule + */ +/** +* @typedef {import('knex').Knex} Knex +*/ +/** + * @typedef {Object} GranulesReport + * @property {number} okCount - The count of granules that are OK. + * @property {number} cumulusCount - The count of granules in Cumulus. + * @property {number} orcaCount - The count of granules in ORCA. + * @property {number} okFilesCount - The count of files that are OK. + * @property {number} cumulusFilesCount - The count of files in Cumulus. + * @property {number} orcaFilesCount - The count of files in ORCA. + * @property {number} conflictFilesCount - The count of files with conflicts. + * @property {Array} withConflicts - The list of granules with conflicts. + * @property {Array} onlyInCumulus - The list of granules only in Cumulus. + * @property {Array} onlyInOrca - The list of granules only in ORCA. + */ + +/** @typedef {OrcaReportGranuleObject & PostgresGranuleRecord } CumulusGranule */ + const log = new Logger({ sender: '@api/lambdas/orca-backup-reconciliation-report' }); const fileConflictTypes = { @@ -29,29 +100,43 @@ const fileConflictTypes = { onlyInOrca: 'onlyInOrca', }; -const granuleFields = ['granuleId', 'collectionId', 'provider', 'createdAt', 'updatedAt']; - /** * Fetch orca configuration for all or specified collections * - * @param {Object} recReportParams - input report params - * @param {Object} recReportParams.collectionIds - array of collectionIds - * @returns {Promise} - list of { collectionId, orca configuration } + * @param {EnhancedNormalizedRecReportParams} recReportParams - input report params + * @returns {Promise} - list of { collectionId, orca configuration } */ async function fetchCollectionsConfig(recReportParams) { + const knex = await getKnexClient(); + /** @type {CollectionConfig} */ const collectionsConfig = {}; - const searchParams = convertToESCollectionSearchParams(pick(recReportParams, ['collectionIds'])); - const esCollectionsIterator = new ESSearchQueue( - { ...searchParams, sort_key: ['name', 'version'] }, 'collection', process.env.ES_INDEX - ); - let nextEsItem = await esCollectionsIterator.shift(); - while (nextEsItem) { - const collectionId = constructCollectionId(nextEsItem.name, nextEsItem.version); - const excludedFileExtensions = get(nextEsItem, 'meta.orca.excludedFileExtensions'); - if (excludedFileExtensions) set(collectionsConfig, `${collectionId}.orca.excludedFileExtensions`, excludedFileExtensions); - nextEsItem = await esCollectionsIterator.shift(); // eslint-disable-line no-await-in-loop + const query = knex('collections') + .select('name', 'version', 'meta'); + if (recReportParams.collectionIds) { //TODO typing + const collectionObjects = recReportParams.collectionIds.map((collectionId) => + deconstructCollectionId(collectionId)); + query.where((builder) => { + collectionObjects.forEach(({ name, version }) => { + builder.orWhere((qb) => { + qb.where('name', name).andWhere('version', version); + }); + }); + }); } + const pgCollectionSearchClient = new QuerySearchClient(query, 100); + + /** @type {{ name: string, version: string, meta: Object }} */ + // @ts-ignore TODO: Ticket CUMULUS-3887 filed to resolve + let nextPgItem = await pgCollectionSearchClient.shift(); + while (nextPgItem) { + const collectionId = constructCollectionId(nextPgItem.name, nextPgItem.version); + const excludedFileExtensions = get(nextPgItem, 'meta.orca.excludedFileExtensions'); + if (excludedFileExtensions) set(collectionsConfig, `${collectionId}.orca.excludedFileExtensions`, excludedFileExtensions); + /** @type {{ name: string, version: string, meta: Object }} */ + // @ts-ignore TODO: Ticket CUMULUS-3887 filed to resolve + nextPgItem = await pgCollectionSearchClient.shift(); // eslint-disable-line no-await-in-loop + } return collectionsConfig; } @@ -72,18 +157,28 @@ function shouldFileBeExcludedFromOrca(collectionsConfig, collectionId, fileName) * compare cumulus granule with its orcaGranule if any, and generate report * * @param {Object} params - * @param {Object} params.collectionsConfig - collections configuration - * @param {Object} params.cumulusGranule - cumulus granule + * @param {CollectionConfig} params.collectionsConfig - collections configuration + * @param {CumulusGranule} params.cumulusGranule - cumulus granule * @param {Object} params.orcaGranule - orca granule - * @returns {Object} - discrepency report of the granule + * @returns {GranuleReport} - discrepancy report of the granule */ function getReportForOneGranule({ collectionsConfig, cumulusGranule, orcaGranule }) { + /** @type {GranuleReport} */ const granuleReport = { ok: false, okFilesCount: 0, cumulusFilesCount: 0, orcaFilesCount: 0, - ...pick(cumulusGranule, granuleFields), + ...{ + granuleId: cumulusGranule.granule_id, + collectionId: constructCollectionId( + cumulusGranule.collectionName, + cumulusGranule.collectionVersion + ), + provider: cumulusGranule.providerName, + createdAt: cumulusGranule.created_at.getTime(), + updatedAt: cumulusGranule.updated_at.getTime(), + }, conflictFiles: [], }; @@ -100,6 +195,11 @@ function getReportForOneGranule({ collectionsConfig, cumulusGranule, orcaGranule // if no granule file conflicts, set granuleReport.ok to true // reducer, key: fileName, value: file object with selected fields + /** + * @param {Object} accumulator + * @param {PostgresFileRecord} currentValue + * @returns {Object} + */ const cumulusFileReducer = (accumulator, currentValue) => { const fileName = path.basename(currentValue.key); return ({ @@ -115,7 +215,9 @@ function getReportForOneGranule({ collectionsConfig, cumulusGranule, orcaGranule }); }; - const cumulusFiles = get(cumulusGranule, 'files', []).reduce(cumulusFileReducer, {}); + const cumulusFilesArray = /** @type {PostgresFileRecord[]} */ (get(cumulusGranule, 'files', [])); + const cumulusFiles = cumulusFilesArray.reduce(cumulusFileReducer, {}); + const orcaFiles = get(orcaGranule, 'files', []).reduce(orcaFileReducer, {}); const allFileNames = Object.keys({ ...cumulusFiles, ...orcaFiles }); allFileNames.forEach((fileName) => { @@ -123,9 +225,19 @@ function getReportForOneGranule({ collectionsConfig, cumulusGranule, orcaGranule granuleReport.cumulusFilesCount += 1; granuleReport.orcaFilesCount += 1; - if (!shouldFileBeExcludedFromOrca(collectionsConfig, cumulusGranule.collectionId, fileName)) { + if ( + !shouldFileBeExcludedFromOrca( + collectionsConfig, + constructCollectionId( + cumulusGranule.collectionName, + cumulusGranule.collectionVersion + ), + fileName + ) + ) { granuleReport.okFilesCount += 1; } else { + /** @type {ConflictFile} */ const conflictFile = { fileName, ...cumulusFiles[fileName], @@ -137,7 +249,16 @@ function getReportForOneGranule({ collectionsConfig, cumulusGranule, orcaGranule } else if (cumulusFiles[fileName] && orcaFiles[fileName] === undefined) { granuleReport.cumulusFilesCount += 1; - if (shouldFileBeExcludedFromOrca(collectionsConfig, cumulusGranule.collectionId, fileName)) { + if ( + shouldFileBeExcludedFromOrca( + collectionsConfig, + constructCollectionId( + cumulusGranule.collectionName, + cumulusGranule.collectionVersion + ), + fileName + ) + ) { granuleReport.okFilesCount += 1; } else { const conflictFile = { @@ -184,10 +305,39 @@ function constructOrcaOnlyGranuleForReport(orcaGranule) { return granule; } -function addGranuleToReport({ granulesReport, collectionsConfig, cumulusGranule, orcaGranule }) { +/** + * Adds a granule to the reconciliation report object + * + * @param {Object} params - The parameters for the function. + * @param {GranulesReport} params.granulesReport - The report object to update. + * @param {CollectionConfig} params.collectionsConfig - The collections configuration. + * @param {CumulusGranule} params.cumulusGranule - The Cumulus granule to add to the report. + * @param {Object} [params.orcaGranule] - The ORCA granule to compare against (optional). + * @param {Knex} params.knex - The Knex database connection. + * @returns {Promise} The updated granules report. + * @throws {Error} If cumulusGranule is not defined. + */ +async function addGranuleToReport({ + granulesReport, + collectionsConfig, + cumulusGranule, + orcaGranule, + knex, +}) { + if (!cumulusGranule) { + throw new Error('cumulusGranule must be defined to add to the orca report'); + } + const modifiedCumulusGranule = { ...cumulusGranule }; + + modifiedCumulusGranule.files = await filePgModel.search(knex, { + granule_cumulus_id: cumulusGranule.cumulus_id, + }); + /* eslint-disable no-param-reassign */ const granReport = getReportForOneGranule({ - collectionsConfig, cumulusGranule, orcaGranule, + collectionsConfig, + cumulusGranule: modifiedCumulusGranule, + orcaGranule, }); if (granReport.ok) { @@ -208,7 +358,7 @@ function addGranuleToReport({ granulesReport, collectionsConfig, cumulusGranule, /** * Compare the granule holdings in Cumulus with ORCA * - * @param {Object} recReportParams - lambda's input filtering parameters + * @param {EnhancedNormalizedRecReportParams} recReportParams - input report params * @returns {Promise} an object with the okCount, onlyInCumulus, onlyInOrca * and withConfilcts */ @@ -221,6 +371,7 @@ async function orcaReconciliationReportForGranules(recReportParams) { // Report granules only in cumulus // Report granules only in orca log.info(`orcaReconciliationReportForGranules ${JSON.stringify(recReportParams)}`); + /** @type {GranulesReport} */ const granulesReport = { okCount: 0, cumulusCount: 0, @@ -235,17 +386,23 @@ async function orcaReconciliationReportForGranules(recReportParams) { }; const collectionsConfig = await fetchCollectionsConfig(recReportParams); - log.debug(`fetchESCollections returned ${JSON.stringify(collectionsConfig)}`); - - const esSearchParams = convertToESGranuleSearchParamsWithCreatedAtRange(recReportParams); - log.debug(`Create ES granule iterator with ${JSON.stringify(esSearchParams)}`); - const esGranulesIterator = new ESSearchQueue( - { - ...esSearchParams, - sort_key: ['granuleId', 'collectionId'], - }, - 'granule', - process.env.ES_INDEX + log.debug(`fetchCollections returned ${JSON.stringify(collectionsConfig)}`); + + const knex = await getKnexClient(); + const searchParams = convertToDBGranuleSearchParams(recReportParams); + + const granulesSearchQuery = getGranulesByApiPropertiesQuery({ + knex, + searchParams, + sortByFields: ['granule_id', 'collectionName', 'collectionVersion'], + temporalBoundByCreatedAt: true, + }); + + log.debug(`Create PG granule iterator with ${granulesSearchQuery}`); + + const pgGranulesIterator = new QuerySearchClient( + granulesSearchQuery, + 100 // arbitrary limit on how items are fetched at once ); const orcaSearchParams = convertToOrcaGranuleSearchParams(recReportParams); @@ -253,22 +410,30 @@ async function orcaReconciliationReportForGranules(recReportParams) { const orcaGranulesIterator = new ORCASearchCatalogQueue(orcaSearchParams); try { + /** @type {[CumulusGranule, any]} */ + // @ts-ignore TODO: Ticket CUMULUS-3887 filed to resolve let [nextCumulusItem, nextOrcaItem] = await Promise.all( - [esGranulesIterator.peek(), orcaGranulesIterator.peek()] + [ + /** @type CumulusGranule */ + pgGranulesIterator.peek(), + orcaGranulesIterator.peek(), + ] ); while (nextCumulusItem && nextOrcaItem) { - const nextCumulusId = `${nextCumulusItem.granuleId}:${nextCumulusItem.collectionId}`; + const nextCumulusId = `${nextCumulusItem.granule_id}:${constructCollectionId(nextCumulusItem.collectionName, nextCumulusItem.collectionVersion)}`; const nextOrcaId = `${nextOrcaItem.id}:${nextOrcaItem.collectionId}`; if (nextCumulusId < nextOrcaId) { // Found an item that is only in Cumulus and not in ORCA. - addGranuleToReport({ + // eslint-disable-next-line no-await-in-loop + await addGranuleToReport({ granulesReport, collectionsConfig, cumulusGranule: nextCumulusItem, + knex, }); granulesReport.cumulusCount += 1; - await esGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + await pgGranulesIterator.shift(); // eslint-disable-line no-await-in-loop } else if (nextCumulusId > nextOrcaId) { // Found an item that is only in ORCA and not in Cumulus granulesReport.onlyInOrca.push(constructOrcaOnlyGranuleForReport(nextOrcaItem)); @@ -277,29 +442,36 @@ async function orcaReconciliationReportForGranules(recReportParams) { } else { // Found an item that is in both ORCA and Cumulus database // Check if the granule (files) should be in orca, and act accordingly - addGranuleToReport({ + // eslint-disable-next-line no-await-in-loop + await addGranuleToReport({ granulesReport, collectionsConfig, cumulusGranule: nextCumulusItem, orcaGranule: nextOrcaItem, + knex, }); granulesReport.cumulusCount += 1; granulesReport.orcaCount += 1; - await esGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + await pgGranulesIterator.shift(); // eslint-disable-line no-await-in-loop await orcaGranulesIterator.shift(); // eslint-disable-line no-await-in-loop } - - [nextCumulusItem, nextOrcaItem] = await Promise.all([esGranulesIterator.peek(), orcaGranulesIterator.peek()]); // eslint-disable-line max-len, no-await-in-loop + /** @type {[CumulusGranule, any]} */ + // @ts-ignore TODO: Ticket CUMULUS-3887 filed to resolve + [nextCumulusItem, nextOrcaItem] = await Promise.all([pgGranulesIterator.peek(), orcaGranulesIterator.peek()]); // eslint-disable-line max-len, no-await-in-loop } // Add any remaining cumulus items to the report - while (await esGranulesIterator.peek()) { // eslint-disable-line no-await-in-loop - const cumulusItem = await esGranulesIterator.shift(); // eslint-disable-line no-await-in-loop + while (await pgGranulesIterator.peek()) { // eslint-disable-line no-await-in-loop + /** @type {CumulusGranule} */ + // @ts-ignore TODO: Ticket CUMULUS-3887 filed to resolve + const cumulusItem = await pgGranulesIterator.shift(); // eslint-disable-line no-await-in-loop // Found an item that is only in Cumulus database and not in ORCA. - addGranuleToReport({ + // eslint-disable-next-line no-await-in-loop + await addGranuleToReport({ granulesReport, collectionsConfig, cumulusGranule: cumulusItem, + knex, }); granulesReport.cumulusCount += 1; } @@ -327,18 +499,8 @@ async function orcaReconciliationReportForGranules(recReportParams) { /** * Create an ORCA Backup Reconciliation report and save it to S3 * - * @param {Object} recReportParams - params - * @param {Object} recReportParams.collectionIds - array of collectionIds - * @param {Object} recReportParams.providers - array of providers - * @param {Object} recReportParams.granuleIds - array of granuleIds - * @param {Object} recReportParams.reportType - the report type - * @param {moment} recReportParams.createStartTime - when the report creation was begun - * @param {moment} recReportParams.endTimestamp - ending report datetime ISO Timestamp - * @param {string} recReportParams.reportKey - the s3 report key - * @param {string} recReportParams.stackName - the name of the CUMULUS stack - * @param {moment} recReportParams.startTimestamp - beginning report datetime ISO timestamp - * @param {string} recReportParams.systemBucket - the name of the CUMULUS system bucket - * @returns {Promise} a Promise that resolves when the report has been + * @param {EnhancedNormalizedRecReportParams} recReportParams - params + * @returns {Promise} a Promise that resolves when the report has been * uploaded to S3 */ async function createOrcaBackupReconciliationReport(recReportParams) { @@ -399,7 +561,7 @@ async function createOrcaBackupReconciliationReport(recReportParams) { report.status = 'SUCCESS'; // Write the full report to S3 - return s3().putObject({ + await s3().putObject({ Bucket: systemBucket, Key: reportKey, Body: JSON.stringify(report, undefined, 2), diff --git a/packages/api/lambdas/sf-event-sqs-to-db-records/index.js b/packages/api/lambdas/sf-event-sqs-to-db-records/index.js index e1609c90bc9..411079ae937 100644 --- a/packages/api/lambdas/sf-event-sqs-to-db-records/index.js +++ b/packages/api/lambdas/sf-event-sqs-to-db-records/index.js @@ -52,14 +52,12 @@ const log = new Logger({ sender: '@cumulus/api/lambdas/sf-event-sqs-to-db-record * @param {Object} params * @param {Object} params.cumulusMessage - Cumulus workflow message * @param {Knex} params.knex - Knex client - * @param {EsClient} params.esClient - Elasticsearch client * @param {Object} [params.testOverrides] * Optional override/mock object used for testing */ const writeRecords = async ({ cumulusMessage, knex, - esClient, testOverrides = {}, }) => { const messageCollectionNameVersion = getCollectionNameAndVersionFromMessage(cumulusMessage); @@ -103,7 +101,6 @@ const writeRecords = async ({ asyncOperationCumulusId, parentExecutionCumulusId, knex, - esClient, }); const providerCumulusId = await getMessageProviderCumulusId(cumulusMessage, knex); @@ -114,13 +111,11 @@ const writeRecords = async ({ providerCumulusId, knex, executionCumulusId, - esClient, }); return writeGranulesFromMessage({ cumulusMessage, executionCumulusId, - esClient, knex, testOverrides, }); diff --git a/packages/api/lambdas/sf-event-sqs-to-db-records/write-pdr.js b/packages/api/lambdas/sf-event-sqs-to-db-records/write-pdr.js index 26fb565ac0e..262b7377a5d 100644 --- a/packages/api/lambdas/sf-event-sqs-to-db-records/write-pdr.js +++ b/packages/api/lambdas/sf-event-sqs-to-db-records/write-pdr.js @@ -5,8 +5,6 @@ const { PdrPgModel, translatePostgresPdrToApiPdr, } = require('@cumulus/db'); -const { upsertPdr } = require('@cumulus/es-client/indexer'); -const { getEsClient } = require('@cumulus/es-client/search'); const { getMessagePdrName, messageHasPdr, @@ -14,7 +12,6 @@ const { getMessagePdrPANSent, getMessagePdrPANMessage, getPdrPercentCompletion, - generatePdrApiRecordFromMessage, } = require('@cumulus/message/PDRs'); const { getMetaStatus, @@ -87,23 +84,6 @@ const writePdrViaTransaction = async ({ return pdr; }; -const writePdrToEs = async (params) => { - const { - cumulusMessage, - updatedAt = Date.now(), - esClient = await getEsClient(), - } = params; - const pdrApiRecord = generatePdrApiRecordFromMessage(cumulusMessage, updatedAt); - if (!pdrApiRecord) { - return; - } - await upsertPdr({ - esClient, - updates: pdrApiRecord, - index: process.env.ES_INDEX, - }); -}; - const writePdr = async ({ cumulusMessage, collectionCumulusId, @@ -111,7 +91,6 @@ const writePdr = async ({ executionCumulusId, knex, updatedAt = Date.now(), - esClient, }) => { let pgPdr; // If there is no PDR in the message, then there's nothing to do here, which is fine @@ -133,11 +112,6 @@ const writePdr = async ({ executionCumulusId, updatedAt, }); - await writePdrToEs({ - cumulusMessage, - updatedAt, - esClient, - }); return pgPdr.cumulus_id; }); const pdrToPublish = await translatePostgresPdrToApiPdr(pgPdr, knex); @@ -149,5 +123,4 @@ module.exports = { generatePdrRecord, writePdrViaTransaction, writePdr, - writePdrToEs, }; diff --git a/packages/api/lib/granules.js b/packages/api/lib/granules.js index 1a9fbfc4e1f..ee0b908d0e8 100644 --- a/packages/api/lib/granules.js +++ b/packages/api/lib/granules.js @@ -234,10 +234,10 @@ function getTotalHits(bodyHits) { } /** - * Returns an array of granules from ElasticSearch query + * Returns an array of granules from an ElasticSearch query * * @param {Object} payload - * @param {string} [payload.index] - ES index to query + * @param {string} [payload.index] - ES index to query (Cloud Metrics) * @param {string} [payload.query] - ES query * @param {Object} [payload.source] - List of IDs to operate on * @param {Object} [payload.testBodyHits] - Optional body.hits for testing. @@ -284,12 +284,12 @@ async function granuleEsQuery({ index, query, source, testBodyHits }) { /** * Return a unique list of granules based on the provided list or the response from the - * query to ES using the provided query and index. + * query to ES (Cloud Metrics) using the provided query and index. * * @param {Object} payload * @param {Object} [payload.granules] - Optional list of granules with granuleId and collectionId - * @param {Object} [payload.query] - Optional parameter of query to send to ES - * @param {string} [payload.index] - Optional parameter of ES index to query. + * @param {Object} [payload.query] - Optional parameter of query to send to ES (Cloud Metrics) + * @param {string} [payload.index] - Optional parameter of ES index to query (Cloud Metrics). * Must exist if payload.query exists. * @returns {Promise>} */ @@ -297,7 +297,7 @@ async function getGranulesForPayload(payload) { const { granules, index, query } = payload; const queryGranules = granules || []; - // query ElasticSearch if needed + // query ElasticSearch (Cloud Metrics) if needed if (queryGranules.length === 0 && query) { log.info('No granules detected. Searching for granules in ElasticSearch.'); diff --git a/packages/api/lib/mmt.js b/packages/api/lib/mmt.js index d37ddc4db77..68581bcc2f0 100644 --- a/packages/api/lib/mmt.js +++ b/packages/api/lib/mmt.js @@ -42,16 +42,16 @@ const buildMMTLink = (conceptId, cmrEnv = process.env.CMR_ENVIRONMENT) => { }; /** - * Updates the Collection query results from ES with an MMTLink when the + * Updates the Collection query results with a MMTLink when the * matching CMR entry contains a collection_id. * - * @param {Array} esResults - collection query results from Cumulus' elasticsearch + * @param {Array} queryResults - collection query results from Cumulus DB * @param {Array} cmrEntries - cmr response feed entry that should match the * results collections - * @returns {Array} - Array of shallow clones of esResults objects with + * @returns {Array} - Array of shallow clones of queryResults objects with * MMTLinks added to them */ -const updateResponseWithMMT = (esResults, cmrEntries) => esResults.map((res) => { +const updateResponseWithMMT = (queryResults, cmrEntries) => queryResults.map((res) => { const matchedCmr = cmrEntries.filter( (entry) => entry.short_name === res.name && entry.version_id === res.version ); @@ -61,7 +61,7 @@ const updateResponseWithMMT = (esResults, cmrEntries) => esResults.map((res) => }); /** - * Simplifies and transforms The returned ES results from a collection query + * Simplifies and transforms the results from a collection query * into a list of objects suitable for a compound call to CMR to retrieve * collection_id information. * Transforms each object in the results array into an new object. @@ -69,7 +69,7 @@ const updateResponseWithMMT = (esResults, cmrEntries) => esResults.map((res) => * inputObject.version => outputObject.version * all other input object keys are dropped. * - * @param {Object} results - The elasticsearch results array returned from either + * @param {Object} results - The results array returned from either * Collection.query() or Collection.queryCollectionsWithActiveGranules() * @returns {Arary} - list of Objects with two keys (short_name and version). */ @@ -80,10 +80,10 @@ const parseResults = (results) => })); /** - * parses the elasticsearch collection lists and for each result inserts a "MMTLink" + * parses the query collection lists and for each result inserts a "MMTLink" * into the collection object. * - * @param {Object} inputResponse - an elasticsearch reponse returned from either + * @param {Object} inputResponse - a reponse returned from either * Collection.query() or Collection.queryCollectionsWithActiveGranules() * @returns {Object} a copy of input response object where each collection * has been updated to include a link to the Metadata Management Tool diff --git a/packages/api/lib/orca.js b/packages/api/lib/orca.js index dcc22aa148f..ea075c0cc7b 100644 --- a/packages/api/lib/orca.js +++ b/packages/api/lib/orca.js @@ -85,7 +85,7 @@ const getOrcaRecoveryStatusByGranuleIdAndCollection = async (granuleId, collecti /** * add recovery status for each granule in the granule list response * - * @param {Object} inputResponse - an elasticsearch response returned from granules query + * @param {Object} inputResponse - a response returned from a granules query * @returns {Object} a copy of input response object where each granule * has been updated to include orca recovery status */ diff --git a/packages/api/lib/reconciliationReport-types.js b/packages/api/lib/reconciliationReport-types.js new file mode 100644 index 00000000000..16e3b5096ef --- /dev/null +++ b/packages/api/lib/reconciliationReport-types.js @@ -0,0 +1,19 @@ +/** + * @typedef {Object} ReportHeader + * @property {string | undefined} collectionId - The collection ID. + * @property {string | string[] | undefined} collectionIds - The collection IDs. + * @property {string | undefined} createEndTime - The end time of the report creation. + * @property {string} createStartTime - The start time of the report creation. + * @property {string | undefined} error - Any error that occurred. + * @property {string | undefined} granuleId - The granule ID. + * @property {string | string[] | undefined} granuleIds - The granule IDs. + * @property {string | string[] | undefined} provider - The provider. + * @property {string | string[] | undefined} providers - The providers. + * @property {string | undefined} location - The location. + * @property {string | undefined} reportEndTime - The end time of the report. + * @property {string | undefined} reportStartTime - The start time of the report. + * @property {string} reportType - The type of the report. + * @property {string} status - The status of the report. + */ + +module.exports = {}; diff --git a/packages/api/lib/reconciliationReport.js b/packages/api/lib/reconciliationReport.js index 328ce25cfd9..f063d3112ad 100644 --- a/packages/api/lib/reconciliationReport.js +++ b/packages/api/lib/reconciliationReport.js @@ -1,7 +1,6 @@ -'use strict'; +//@ts-check -const isEqual = require('lodash/isEqual'); -const omit = require('lodash/omit'); +'use strict'; const { removeNilProperties } = require('@cumulus/common/util'); const { constructCollectionId, deconstructCollectionId } = require('@cumulus/message/Collections'); @@ -9,6 +8,14 @@ const Logger = require('@cumulus/logger'); const log = new Logger({ sender: '@api/lambdas/create-reconciliation-report' }); +/** + * @typedef {import('../lib/types').RecReportParams } RecReportParams + * @typedef {import('../lib/types').EnhancedNormalizedRecReportParams } + * EnhancedNormalizedRecReportParams + * @typedef {import('../lib/types').NormalizedRecReportParams } NormalizedRecReportParams + * @typedef {import('./reconciliationReport-types').ReportHeader } ReportHeader + */ + /** * Extra search params to add to the cmrGranules searchConceptQueue * @@ -23,20 +30,9 @@ function cmrGranuleSearchParams(recReportParams) { return []; } -/** - * Prepare a list of collectionIds into an _id__in object - * - * @param {Array} collectionIds - Array of collectionIds in the form 'name___ver' - * @returns {Object} - object that will return the correct terms search when - * passed to the query command. - */ -function searchParamsForCollectionIdArray(collectionIds) { - return { _id__in: collectionIds.join(',') }; -} - /** * @param {string} dateable - any input valid for a JS Date contstructor. - * @returns {number} - primitive value of input date string or undefined, if + * @returns {number | undefined} - primitive value of input date string or undefined, if * input string not convertable. */ function dateToValue(dateable) { @@ -49,34 +45,16 @@ function dateStringToDateOrNull(dateable) { return !Number.isNaN(date.valueOf()) ? date : undefined; } -/** - * - * @param {Object} params - request params to convert to Elasticsearch params - * @returns {Object} object of desired parameters formatted for Elasticsearch collection search - */ -function convertToESCollectionSearchParams(params) { - const { collectionIds, startTimestamp, endTimestamp } = params; - const idsIn = collectionIds - ? searchParamsForCollectionIdArray(collectionIds) - : undefined; - const searchParams = { - updatedAt__from: dateToValue(startTimestamp), - updatedAt__to: dateToValue(endTimestamp), - ...idsIn, - }; - return removeNilProperties(searchParams); -} - /** * convertToDBCollectionSearchObject - Creates Postgres search object from * InternalRecReport Parameters * @param {Object} params - request params to convert to database params - * @param {[Object]} params.collectionIds - List containing single Collection object + * @param {string[]} [params.collectionIds] - List containing single Collection object * multiple or no collections will result in a * search object without a collection object - * @param {moment} params.endTimestamp - ending report datetime ISO Timestamp - * @param {moment} params.startTimestamp - beginning report datetime ISO timestamp - * @returns {[Object]} - array of objects of desired + * @param {string} [params.endTimestamp] - ending report datetime ISO Timestamp + * @param {string} [params.startTimestamp] - beginning report datetime ISO timestamp + * @returns {Object[]} - array of objects of desired * parameters formatted for database collection * search */ @@ -99,30 +77,11 @@ function convertToDBCollectionSearchObject(params) { return searchParams; } -/** - * - * @param {Object} params - request params to convert to Elasticsearch params - * @returns {Object} object of desired parameters formated for Elasticsearch. - */ -function convertToESGranuleSearchParams(params) { - const { collectionIds, granuleIds, providers, startTimestamp, endTimestamp } = params; - const collectionIdIn = collectionIds ? collectionIds.join(',') : undefined; - const granuleIdIn = granuleIds ? granuleIds.join(',') : undefined; - const providerIn = providers ? providers.join(',') : undefined; - return removeNilProperties({ - updatedAt__from: dateToValue(startTimestamp), - updatedAt__to: dateToValue(endTimestamp), - collectionId__in: collectionIdIn, - granuleId__in: granuleIdIn, - provider__in: providerIn, - }); -} - /** * Convert reconciliation report parameters to PostgreSQL database search params. * - * @param {Object} params - request params to convert to database params - * @returns {Object} object of desired parameters formated for database granule search + * @param {EnhancedNormalizedRecReportParams} params - request params to convert to database params + * @returns object of desired parameters formatted for database granule search */ function convertToDBGranuleSearchParams(params) { const { @@ -148,26 +107,10 @@ function convertToDBGranuleSearchParams(params) { return removeNilProperties(searchParams); } -/** - * convert to es search parameters using createdAt for report time range - * - * @param {Object} params - request params to convert to Elasticsearch params - * @returns {Object} object of desired parameters formated for Elasticsearch. - */ -function convertToESGranuleSearchParamsWithCreatedAtRange(params) { - const searchParamsWithUpdatedAt = convertToESGranuleSearchParams(params); - const searchParamsWithCreatedAt = { - createdAt__from: searchParamsWithUpdatedAt.updatedAt__from, - createdAt__to: searchParamsWithUpdatedAt.updatedAt__to, - ...omit(searchParamsWithUpdatedAt, ['updatedAt__from', 'updatedAt__to']), - }; - return removeNilProperties(searchParamsWithCreatedAt); -} - /** * * @param {Object} params - request params to convert to orca params - * @returns {Object} object of desired parameters formated for orca + * @returns {Object} object of desired parameters formatted for orca */ function convertToOrcaGranuleSearchParams(params) { const { collectionIds, granuleIds, providers, startTimestamp, endTimestamp } = params; @@ -183,12 +126,8 @@ function convertToOrcaGranuleSearchParams(params) { /** * create initial report header * - * @param {Object} recReportParams - params - * @param {Object} recReportParams.reportType - the report type - * @param {moment} recReportParams.createStartTime - when the report creation was begun - * @param {moment} recReportParams.endTimestamp - ending report datetime ISO Timestamp - * @param {moment} recReportParams.startTimestamp - beginning report datetime ISO timestamp - * @returns {Object} report header + * @param {EnhancedNormalizedRecReportParams} recReportParams - params + * @returns {ReportHeader} report header */ function initialReportHeader(recReportParams) { const { @@ -241,59 +180,11 @@ function filterDBCollections(collections, recReportParams) { return collections; } -/** - * Compare granules from Elasticsearch and API for deep equality. - * - * @param {Object} esGranule - Granule from Elasticsearch - * @param {Object} apiGranule - API Granule (translated from PostgreSQL) - * @returns {boolean} - */ -function compareEsGranuleAndApiGranule(esGranule, apiGranule) { - // Ignore files in initial comparison so we can ignore file order - // in comparison - const fieldsIgnored = ['timestamp', 'updatedAt', 'files']; - // "dataType" and "version" fields do not exist in the PostgreSQL database - // granules table which is now the source of truth - const esFieldsIgnored = [...fieldsIgnored, 'dataType', 'version']; - const granulesAreEqual = isEqual( - omit(esGranule, esFieldsIgnored), - omit(apiGranule, fieldsIgnored) - ); - - if (granulesAreEqual === false) return granulesAreEqual; - - const esGranulesHasFiles = esGranule.files !== undefined; - const apiGranuleHasFiles = apiGranule.files.length !== 0; - - // If neither granule has files, then return the previous equality result - if (!esGranulesHasFiles && !apiGranuleHasFiles) return granulesAreEqual; - // If either ES or PG granule does not have files, but the other granule does - // have files, then the granules don't match, so return false - if ((esGranulesHasFiles && !apiGranuleHasFiles) - || (!esGranulesHasFiles && apiGranuleHasFiles)) { - return false; - } - - // Compare files one-by-one to ignore sort order for comparison - return esGranule.files.every((esFile) => { - const matchingFile = apiGranule.files.find( - (apiFile) => apiFile.bucket === esFile.bucket && apiFile.key === esFile.key - ); - if (!matchingFile) return false; - return isEqual(esFile, matchingFile); - }); -} - module.exports = { cmrGranuleSearchParams, convertToDBCollectionSearchObject, convertToDBGranuleSearchParams, - convertToESCollectionSearchParams, - convertToESGranuleSearchParams, - convertToESGranuleSearchParamsWithCreatedAtRange, convertToOrcaGranuleSearchParams, filterDBCollections, initialReportHeader, - searchParamsForCollectionIdArray, - compareEsGranuleAndApiGranule, }; diff --git a/packages/api/lib/reconciliationReport/normalizeEvent.js b/packages/api/lib/reconciliationReport/normalizeEvent.js index 88cd2283df1..b623f7e4a51 100644 --- a/packages/api/lib/reconciliationReport/normalizeEvent.js +++ b/packages/api/lib/reconciliationReport/normalizeEvent.js @@ -1,15 +1,22 @@ +//@ts-check + 'use strict'; /*eslint prefer-const: ["error", {"destructuring": "all"}]*/ const isString = require('lodash/isString'); const { removeNilProperties } = require('@cumulus/common/util'); -const { InvalidArgument } = require('@cumulus/errors'); +const { InvalidArgument, MissingRequiredArgument } = require('@cumulus/errors'); + +/** + * @typedef {import('../types').RecReportParams } RecReportParams + * @typedef {import('../types').NormalizedRecReportParams } NormalizedRecReportParams + */ /** * ensures input reportType can be handled by the lambda code. * * @param {string} reportType - * @returns {undefined} - if reportType is valid + * @returns {void} - if reportType is valid * @throws {InvalidArgument} - otherwise */ function validateReportType(reportType) { @@ -31,7 +38,7 @@ function validateReportType(reportType) { /** * Convert input to an ISO timestamp. * @param {any} dateable - any type convertable to JS Date - * @returns {string} - date formated as ISO timestamp; + * @returns {string | undefined} - date formated as ISO timestamp; */ function isoTimestamp(dateable) { if (dateable) { @@ -45,26 +52,19 @@ function isoTimestamp(dateable) { } /** - * Transforms input granuleId into correct parameters for use in the - * Reconciliation Report lambda. - * @param {Array|string} granuleId - list of granule Ids - * @param {Object} modifiedEvent - input event - * @returns {Object} updated input even with correct granuleId and granuleIds values. + * Normalizes the input into an array of granule IDs. + * + * @param {string|string[]|undefined} granuleId - The granule ID or an array of granule IDs. + * @returns {string[]|undefined} An array of granule IDs, or undefined if no granule ID is provided. */ -function updateGranuleIds(granuleId, modifiedEvent) { - let returnEvent = { ...modifiedEvent }; - if (granuleId) { - // transform input granuleId into an array on granuleIds - const granuleIds = isString(granuleId) ? [granuleId] : granuleId; - returnEvent = { ...modifiedEvent, granuleIds }; - } - return returnEvent; +function generateGranuleIds(granuleId) { + return granuleId ? (isString(granuleId) ? [granuleId] : granuleId) : undefined; } /** * Transforms input collectionId into correct parameters for use in the * Reconciliation Report lambda. - * @param {Array|string} collectionId - list of collection Ids + * @param {string[]|string | undefined} collectionId - list of collection Ids * @param {Object} modifiedEvent - input event * @returns {Object} updated input even with correct collectionId and collectionIds values. */ @@ -78,26 +78,32 @@ function updateCollectionIds(collectionId, modifiedEvent) { return returnEvent; } -function updateProviders(provider, modifiedEvent) { - let returnEvent = { ...modifiedEvent }; - if (provider) { - // transform input provider into an array on providers - const providers = isString(provider) ? [provider] : provider; - returnEvent = { ...modifiedEvent, providers }; - } - return returnEvent; +/** + * Normalizes the input provider into an array of providers. + * + * @param {string|string[]|undefined} provider - The provider or list of providers. + * @returns {string[]|undefined} An array of providers, or undefined if no provider is provided. + */ +function generateProviders(provider) { + return provider ? (isString(provider) ? [provider] : provider) : undefined; } /** * Converts input parameters to normalized versions to pass on to the report * functions. Ensures any input dates are formatted as ISO strings. * - * @param {Object} event - input payload - * @returns {Object} - Object with normalized parameters + * @param {RecReportParams} event - input payload + * @returns {NormalizedRecReportParams} - Object with normalized parameters */ function normalizeEvent(event) { const systemBucket = event.systemBucket || process.env.system_bucket; + if (!systemBucket) { + throw new MissingRequiredArgument('systemBucket is required.'); + } const stackName = event.stackName || process.env.stackName; + if (!stackName) { + throw new MissingRequiredArgument('stackName is required.'); + } const startTimestamp = isoTimestamp(event.startTimestamp); const endTimestamp = isoTimestamp(event.endTimestamp); @@ -105,7 +111,11 @@ function normalizeEvent(event) { validateReportType(reportType); let { - collectionIds: anyCollectionIds, collectionId, granuleId, provider, ...modifiedEvent + collectionIds: anyCollectionIds, + collectionId = undefined, + granuleId = undefined, + provider = undefined, + ...modifiedEvent } = { ...event }; if (anyCollectionIds) { throw new InvalidArgument('`collectionIds` is not a valid input key for a reconciliation report, use `collectionId` instead.'); @@ -120,16 +130,16 @@ function normalizeEvent(event) { throw new InvalidArgument(`${reportType} reports cannot be launched with more than one input (granuleId, collectionId, or provider).`); } modifiedEvent = updateCollectionIds(collectionId, modifiedEvent); - modifiedEvent = updateGranuleIds(granuleId, modifiedEvent); - modifiedEvent = updateProviders(provider, modifiedEvent); - return removeNilProperties({ + return (removeNilProperties({ ...modifiedEvent, systemBucket, stackName, startTimestamp, endTimestamp, reportType, - }); + granuleIds: generateGranuleIds(granuleId), + providers: generateProviders(provider), + })); } exports.normalizeEvent = normalizeEvent; diff --git a/packages/api/lib/testUtils.js b/packages/api/lib/testUtils.js index f75f222f3bf..506948315ac 100644 --- a/packages/api/lib/testUtils.js +++ b/packages/api/lib/testUtils.js @@ -19,16 +19,8 @@ const { translateApiExecutionToPostgresExecution, translateApiProviderToPostgresProvider, translateApiRuleToPostgresRuleRaw, - translatePostgresPdrToApiPdr, translatePostgresRuleToApiRule, } = require('@cumulus/db'); -const { - indexProvider, - indexRule, - indexPdr, - indexAsyncOperation, - deleteExecution, -} = require('@cumulus/es-client/indexer'); const { constructCollectionId, } = require('@cumulus/message/Collections'); @@ -238,7 +230,7 @@ function fakeAsyncOperationFactory(params = {}) { taskArn: randomId('arn'), id: uuidv4(), description: randomId('description'), - operationType: 'ES Index', + operationType: 'Reconciliation Report', status: 'SUCCEEDED', createdAt: Date.now() - 180.5 * 1000, updatedAt: Date.now(), @@ -510,8 +502,6 @@ const createProviderTestRecords = async (context, providerParams) => { const { testKnex, providerPgModel, - esClient, - esProviderClient, } = context; const originalProvider = fakeProviderFactory(providerParams); @@ -520,14 +510,9 @@ const createProviderTestRecords = async (context, providerParams) => { const originalPgRecord = await providerPgModel.get( testKnex, { cumulus_id: pgProvider.cumulus_id } ); - await indexProvider(esClient, originalProvider, process.env.ES_INDEX); - const originalEsRecord = await esProviderClient.get( - originalProvider.id - ); return { originalProvider, originalPgRecord, - originalEsRecord, }; }; @@ -538,14 +523,12 @@ const createProviderTestRecords = async (context, providerParams) => { * @param {PostgresRule} - Postgres Rule parameters * * @returns {Object} - * Returns new object consisting of `originalApiRule`, `originalPgRecord, and `originalEsRecord` + * Returns new object consisting of `originalApiRule` and `originalPgRecord` */ const createRuleTestRecords = async (context, ruleParams) => { const { testKnex, rulePgModel, - esClient, - esRulesClient, } = context; const originalRule = fakeRuleRecordFactory(ruleParams); @@ -556,14 +539,10 @@ const createRuleTestRecords = async (context, ruleParams) => { const [originalPgRecord] = await rulePgModel.create(testKnex, pgRuleWithTrigger, '*'); const originalApiRule = await translatePostgresRuleToApiRule(originalPgRecord, testKnex); - await indexRule(esClient, originalApiRule, process.env.ES_INDEX); - const originalEsRecord = await esRulesClient.get( - originalRule.name - ); + return { originalApiRule, originalPgRecord, - originalEsRecord, }; }; @@ -571,8 +550,6 @@ const createPdrTestRecords = async (context, pdrParams = {}) => { const { knex, pdrPgModel, - esClient, - esPdrsClient, collectionCumulusId, providerCumulusId, } = context; @@ -594,14 +571,8 @@ const createPdrTestRecords = async (context, pdrParams = {}) => { const originalPgRecord = await pdrPgModel.get( knex, { cumulus_id: pgPdr.cumulus_id } ); - const originalPdr = await translatePostgresPdrToApiPdr(originalPgRecord, knex); - await indexPdr(esClient, originalPdr, process.env.ES_INDEX); - const originalEsRecord = await esPdrsClient.get( - originalPdr.pdrName - ); return { originalPgRecord, - originalEsRecord, }; }; @@ -627,8 +598,6 @@ const createAsyncOperationTestRecords = async (context) => { const { knex, asyncOperationPgModel, - esClient, - esAsyncOperationClient, } = context; const originalAsyncOperation = fakeAsyncOperationFactory(); @@ -643,32 +612,11 @@ const createAsyncOperationTestRecords = async (context) => { const originalPgRecord = await asyncOperationPgModel.get( knex, { cumulus_id: pgAsyncOperation.cumulus_id } ); - await indexAsyncOperation(esClient, originalAsyncOperation, process.env.ES_INDEX); - const originalEsRecord = await esAsyncOperationClient.get( - originalAsyncOperation.id - ); return { originalPgRecord, - originalEsRecord, }; }; -const cleanupExecutionTestRecords = async (context, { arn }) => { - const { - knex, - executionPgModel, - esClient, - esIndex, - } = context; - - await executionPgModel.delete(knex, { arn }); - await deleteExecution({ - esClient, - arn, - index: esIndex, - }); -}; - module.exports = { createFakeJwtAuthToken, createSqsQueues, @@ -699,6 +647,5 @@ module.exports = { createRuleTestRecords, createPdrTestRecords, createExecutionTestRecords, - cleanupExecutionTestRecords, createAsyncOperationTestRecords, }; diff --git a/packages/api/lib/types.js b/packages/api/lib/types.js new file mode 100644 index 00000000000..2d63fc7decc --- /dev/null +++ b/packages/api/lib/types.js @@ -0,0 +1,46 @@ +/** + * @typedef {Object} NormalizedRecReportParams + * @property {string[]} [collectionIds] - An optional array of collection IDs. + * @property {string[]} [granuleIds] - An optional array of granule IDs. + * @property {string[]} [providers] - An optional array of provider names. + * @property {string} [startTimestamp] - An optional start timestamp for the report. + * @property {string} [endTimestamp] - An optional end timestamp for the report. + * @property {string} [reportType] - An optional type of the report. + * @property {string} [location] + * @property {string} stackName + * @property {string} systemBucket + * @property {string} [status] - Optional granule status filter for report + */ + +/** + * @typedef {Object} EnhancedParams + * @property {Moment.moment} createStartTime - Report creation start time. + * @property {string} reportKey - Key to store report object in S3 + * @property {string} reportType - Type of the report + * @property {Knex} knex - Knex instance + * @property {string} concurrency - Concurrency used in report generation + * @property {string} [location] - Location of the report +*/ + +/** + * @typedef { NormalizedRecReportParams & EnhancedParams} EnhancedNormalizedRecReportParams + */ + +/** + * @typedef {Object} RecReportParams + * @property {string[]} [collectionIds] - An optional array of collection IDs. + * @property {string[]} [granuleIds] - An optional array of granule IDs. + * @property {string[]} [providers] - An optional array of provider names. + * @property {string|Date} [startTimestamp] - An optional start timestamp for the report. + * @property {string|Date} [endTimestamp] - An optional end timestamp for the report. + * @property {string} [reportType] - An optional type of the report. + * @property {boolean} [includeDeleted] - An optional flag to include deleted records. + * @property {boolean} [ignoreFilesConfig] - An optional flag to ignore files configuration. + * @property {string} [bucket] - An optional bucket name for the report. + * @property {string} [stackName] - An optional stack name for the report. + * @property {string} [systemBucket] - An optional system bucket name for the report. + * @property {string} [location] + * @property {string} [status] - Optional granule status filter for report + */ + +module.exports = {}; diff --git a/packages/api/lib/writeRecords/write-execution.js b/packages/api/lib/writeRecords/write-execution.js index d562ddbbf37..0fa24e98211 100644 --- a/packages/api/lib/writeRecords/write-execution.js +++ b/packages/api/lib/writeRecords/write-execution.js @@ -1,3 +1,5 @@ +// @ts-check + const isNil = require('lodash/isNil'); const isUndefined = require('lodash/isUndefined'); const omitBy = require('lodash/omitBy'); @@ -64,7 +66,7 @@ const buildExecutionRecord = ({ const record = { arn, status: getMetaStatus(cumulusMessage), - url: getExecutionUrlFromArn(arn), + url: arn ? getExecutionUrlFromArn(arn) : undefined, cumulus_version: getMessageCumulusVersion(cumulusMessage), tasks: getMessageWorkflowTasks(cumulusMessage), workflow_name: getMessageWorkflowName(cumulusMessage), diff --git a/packages/api/package.json b/packages/api/package.json index c49bf9d0321..4ec0a295359 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -108,6 +108,7 @@ "p-retry": "^2.0.0", "p-wait-for": "^2.0.1", "p-settle": "^4.1.1", + "p-times": "^4.0.0", "querystring": "^0.2.0", "saml2-js": "^4.0.0", "semver": "^7.3.2", diff --git a/packages/api/tests/endpoints/async-operations/test-create-async-operations.js b/packages/api/tests/endpoints/async-operations/test-create-async-operations.js index 8b7a73477a8..e7bf6e5c553 100644 --- a/packages/api/tests/endpoints/async-operations/test-create-async-operations.js +++ b/packages/api/tests/endpoints/async-operations/test-create-async-operations.js @@ -9,11 +9,6 @@ const sinon = require('sinon'); const { s3 } = require('@cumulus/aws-client/services'); const { recursivelyDeleteS3Bucket } = require('@cumulus/aws-client/S3'); const { randomId, randomString } = require('@cumulus/common/test-utils'); -const { Search } = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { localStackConnectionEnv, generateLocalTestDb, @@ -26,8 +21,6 @@ const { const assertions = require('../../../lib/assertions'); const { fakeAsyncOperationFactory } = require('../../../lib/testUtils'); -const { buildFakeExpressResponse } = require('../utils'); -const { post } = require('../../../endpoints/async-operations'); const { createFakeJwtAuthToken, setAuthorizedOAuthUsers, @@ -58,15 +51,6 @@ test.before(async (t) => { t.context.testKnexAdmin = knexAdmin; t.context.asyncOperationPgModel = new AsyncOperationPgModel(); - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esAsyncOperationsClient = new Search( - {}, - 'asyncOperation', - t.context.esIndex - ); - await s3().createBucket({ Bucket: process.env.system_bucket }); const username = randomString(); @@ -83,7 +67,6 @@ test.before(async (t) => { test.after.always(async (t) => { await t.context.accessTokenModel.deleteTable().catch(noop); - await cleanupTestIndex(t.context); await recursivelyDeleteS3Bucket(process.env.system_bucket); await destroyLocalTestDb({ knex: t.context.testKnex, @@ -117,7 +100,7 @@ test('POST with an invalid access token returns an unauthorized response', async assertions.isInvalidAccessTokenResponse(t, response); }); -test('POST creates a new async operation in all data stores', async (t) => { +test('POST creates and stores expected new async operation record', async (t) => { const { asyncOperationPgModel, jwtAuthToken } = t.context; const asyncOperation = fakeAsyncOperationFactory({ output: JSON.stringify({ age: 59 }), @@ -151,14 +134,9 @@ test('POST creates a new async operation in all data stores', async (t) => { omit(pgAsyncOperation, omitList) ); t.deepEqual(asyncOperationPgRecord.output, pgAsyncOperation.output); - - const esRecord = await t.context.esAsyncOperationsClient.get( - asyncOperation.id - ); - t.like(esRecord, record); }); -test('POST creates a new async operation in PostgreSQL/Elasticsearch with correct timestamps', async (t) => { +test('POST creates a new async operation record with correct timestamps', async (t) => { const { asyncOperationPgModel, jwtAuthToken } = t.context; const asyncOperation = fakeAsyncOperationFactory({ output: JSON.stringify({ age: 59 }), @@ -184,12 +162,8 @@ test('POST creates a new async operation in PostgreSQL/Elasticsearch with correc t.true(apiRecord.createdAt > asyncOperation.createdAt); t.true(apiRecord.updatedAt > asyncOperation.updatedAt); - const esRecord = await t.context.esAsyncOperationsClient.get(asyncOperation.id); - t.is(asyncOperationPgRecord.created_at.getTime(), record.createdAt); t.is(asyncOperationPgRecord.updated_at.getTime(), record.updatedAt); - t.is(asyncOperationPgRecord.created_at.getTime(), esRecord.createdAt); - t.is(asyncOperationPgRecord.updated_at.getTime(), esRecord.updatedAt); }); test('POST returns a 409 error if the async operation already exists in PostgreSQL', async (t) => { @@ -269,33 +243,3 @@ test('POST returns a 400 response if invalid JSON provided', async (t) => { t.is(error.error, 'Bad Request'); t.is(error.message, 'Async Operations require an ID'); }); - -test('post() does not write to PostgreSQL if writing to Elasticsearch fails', async (t) => { - const asyncOperation = fakeAsyncOperationFactory({ - output: JSON.stringify({ age: 59 }), - }); - const fakeEsClient = { - client: { - index: () => Promise.reject(new Error('something bad')), - }, - }; - - const expressRequest = { - body: asyncOperation, - testContext: { - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await post(expressRequest, response); - - t.true(response.boom.badImplementation.calledWithMatch('something bad')); - - t.false( - await t.context.asyncOperationPgModel.exists(t.context.testKnex, { - id: asyncOperation.id, - }) - ); -}); diff --git a/packages/api/tests/endpoints/async-operations/test-endpoints-async-operations.js b/packages/api/tests/endpoints/async-operations/test-endpoints-async-operations.js index dcfd4703cf4..87f75646a6e 100644 --- a/packages/api/tests/endpoints/async-operations/test-endpoints-async-operations.js +++ b/packages/api/tests/endpoints/async-operations/test-endpoints-async-operations.js @@ -17,12 +17,6 @@ const { translateApiAsyncOperationToPostgresAsyncOperation, migrationDir, } = require('@cumulus/db'); -const { Search } = require('@cumulus/es-client/search'); -const indexer = require('@cumulus/es-client/indexer'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { fakeAsyncOperationFactory } = require('../../../lib/testUtils'); const { @@ -36,7 +30,6 @@ const { setAuthorizedOAuthUsers, createAsyncOperationTestRecords, } = require('../../../lib/testUtils'); -const { buildFakeExpressResponse } = require('../utils'); process.env.stackName = randomString(); process.env.system_bucket = randomString(); @@ -64,15 +57,6 @@ test.before(async (t) => { t.context.asyncOperationPgModel = new AsyncOperationPgModel(); - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esAsyncOperationClient = new Search( - {}, - 'asyncOperation', - t.context.esIndex - ); - await s3().createBucket({ Bucket: process.env.system_bucket }); const username = randomString(); @@ -92,21 +76,17 @@ test.after.always(async (t) => { knexAdmin: t.context.testKnexAdmin, testDbName, }); - await cleanupTestIndex(t.context); }); test.serial('GET /asyncOperations returns a list of operations', async (t) => { - const { esClient, esIndex } = t.context; const asyncOperation1 = fakeAsyncOperationFactory(); const asyncOperation2 = fakeAsyncOperationFactory(); const asyncOpPgRecord1 = translateApiAsyncOperationToPostgresAsyncOperation(asyncOperation1); await t.context.asyncOperationPgModel.create(t.context.knex, asyncOpPgRecord1); - await indexer.indexAsyncOperation(esClient, asyncOperation1, esIndex); const asyncOpPgRecord2 = translateApiAsyncOperationToPostgresAsyncOperation(asyncOperation2); await t.context.asyncOperationPgModel.create(t.context.knex, asyncOpPgRecord2); - await indexer.indexAsyncOperation(esClient, asyncOperation2, esIndex); const response = await request(app) .get('/asyncOperations') @@ -134,19 +114,15 @@ test.serial('GET /asyncOperations returns a list of operations', async (t) => { }); test.serial('GET /asyncOperations with a timestamp parameter returns a list of filtered results', async (t) => { - const { esClient, esIndex } = t.context; const firstDate = Date.now(); const asyncOperation1 = fakeAsyncOperationFactory(); - const asyncOperation2 = fakeAsyncOperationFactory(); const asyncOpPgRecord1 = translateApiAsyncOperationToPostgresAsyncOperation(asyncOperation1); await t.context.asyncOperationPgModel.create(t.context.knex, asyncOpPgRecord1); - await indexer.indexAsyncOperation(esClient, asyncOperation1, esIndex); const secondDate = Date.now(); - + const asyncOperation2 = fakeAsyncOperationFactory(); const asyncOpPgRecord2 = translateApiAsyncOperationToPostgresAsyncOperation(asyncOperation2); await t.context.asyncOperationPgModel.create(t.context.knex, asyncOpPgRecord2); - await indexer.indexAsyncOperation(esClient, asyncOperation2, esIndex); const response1 = await request(app) .get(`/asyncOperations?timestamp__from=${firstDate}`) @@ -225,7 +201,7 @@ test('del() returns a 401 bad request if id is not provided', async (t) => { t.true(fakeResponse.boom.badRequest.called); }); -test('DELETE returns a 404 if PostgreSQL and Elasticsearch async operation cannot be found', async (t) => { +test('DELETE returns a 404 if PostgreSQL async operation cannot be found', async (t) => { const nonExistentAsyncOperation = fakeAsyncOperationFactory(); const response = await request(app) .delete(`/asyncOperations/${nonExistentAsyncOperation.id}`) @@ -235,76 +211,7 @@ test('DELETE returns a 404 if PostgreSQL and Elasticsearch async operation canno t.is(response.body.message, 'No record found'); }); -test('DELETE deletes async operation successfully if it exists in PostgreSQL but not Elasticsearch', async (t) => { - const { - asyncOperationPgModel, - esAsyncOperationClient, - knex, - } = t.context; - - const originalAsyncOperation = fakeAsyncOperationFactory(); - const insertPgRecord = await translateApiAsyncOperationToPostgresAsyncOperation( - originalAsyncOperation, - knex - ); - const id = insertPgRecord.id; - await asyncOperationPgModel.create( - knex, - insertPgRecord - ); - t.true( - await asyncOperationPgModel.exists(knex, { id }) - ); - - const response = await request(app) - .delete(`/asyncOperations/${id}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - const { message } = response.body; - - t.is(message, 'Record deleted'); - t.false( - await asyncOperationPgModel.exists(knex, { id }) - ); - t.false(await esAsyncOperationClient.exists( - id - )); -}); - -test('DELETE deletes async operation successfully if it exists Elasticsearch but not PostgreSQL', async (t) => { - const { - asyncOperationPgModel, - esAsyncOperationClient, - esClient, - esIndex, - knex, - } = t.context; - - const originalAsyncOperation = fakeAsyncOperationFactory(); - const id = originalAsyncOperation.id; - await indexer.indexAsyncOperation(esClient, originalAsyncOperation, esIndex); - t.false( - await asyncOperationPgModel.exists(knex, { id }) - ); - t.true( - await esAsyncOperationClient.exists(id) - ); - - const response = await request(app) - .delete(`/asyncOperations/${id}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - const { message } = response.body; - - t.is(message, 'Record deleted'); - t.false( - await esAsyncOperationClient.exists(id) - ); -}); - -test('DELETE deletes the async operation from all data stores', async (t) => { +test('DELETE deletes the async operation from the database', async (t) => { const { originalPgRecord, } = await createAsyncOperationTestRecords(t.context); @@ -326,93 +233,4 @@ test('DELETE deletes the async operation from all data stores', async (t) => { const dbRecords = await t.context.asyncOperationPgModel .search(t.context.knex, { id }); t.is(dbRecords.length, 0); - t.false(await t.context.esAsyncOperationClient.exists( - id - )); -}); - -test('del() does not remove from Elasticsearch if removing from PostgreSQL fails', async (t) => { - const { - originalPgRecord, - } = await createAsyncOperationTestRecords(t.context); - const { id } = originalPgRecord; - - const fakeAsyncOperationPgModel = { - delete: () => { - throw new Error('PG something bad'); - }, - get: () => Promise.resolve(originalPgRecord), - }; - - const expressRequest = { - params: { - id, - }, - testContext: { - knex: t.context.knex, - asyncOperationPgModel: fakeAsyncOperationPgModel, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - del(expressRequest, response), - { message: 'PG something bad' } - ); - - t.true( - await t.context.asyncOperationPgModel.exists(t.context.knex, { - id, - }) - ); - t.true( - await t.context.esAsyncOperationClient.exists( - id - ) - ); -}); - -test('del() does not remove from PostgreSQL if removing from Elasticsearch fails', async (t) => { - const { - originalPgRecord, - } = await createAsyncOperationTestRecords(t.context); - const { id } = originalPgRecord; - - const fakeEsClient = { - initializeEsClient: () => Promise.resolve(), - client: { - delete: () => { - throw new Error('ES something bad'); - }, - }, - }; - - const expressRequest = { - params: { - id, - }, - testContext: { - knex: t.context.knex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - del(expressRequest, response), - { message: 'ES something bad' } - ); - - t.true( - await t.context.asyncOperationPgModel.exists(t.context.knex, { - id, - }) - ); - t.true( - await t.context.esAsyncOperationClient.exists( - id - ) - ); }); diff --git a/packages/api/tests/endpoints/granules/test-searchafter-10k.js b/packages/api/tests/endpoints/granules/test-searchafter-10k.js index ccc927c01ee..8966033d661 100644 --- a/packages/api/tests/endpoints/granules/test-searchafter-10k.js +++ b/packages/api/tests/endpoints/granules/test-searchafter-10k.js @@ -2,10 +2,9 @@ const test = require('ava'); const request = require('supertest'); -const { randomId } = require('@cumulus/common/test-utils'); -const { getEsClient } = require('@cumulus/es-client/search'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const { loadGranules, granuleFactory } = require('@cumulus/es-client/tests/helpers/helpers'); +const cryptoRandomString = require('crypto-random-string'); + +const { randomId, randomString } = require('@cumulus/common/test-utils'); process.env.AccessTokensTable = randomId('token'); process.env.stackName = randomId('stackname'); @@ -13,70 +12,80 @@ process.env.system_bucket = randomId('system-bucket'); process.env.TOKEN_SECRET = randomId('secret'); process.env.backgroundQueueUrl = randomId('backgroundQueueUrl'); +const { + CollectionPgModel, + destroyLocalTestDb, + fakeCollectionRecordFactory, + fakeGranuleRecordFactory, + generateLocalTestDb, + GranulePgModel, + localStackConnectionEnv, + migrationDir, +} = require('@cumulus/db'); + // import the express app after setting the env variables const { app } = require('../../../app'); test.before(async (t) => { + const concurrency = 200; + const granuleTotal = 10001; + const { default: pTimes } = await import('p-times'); process.env.NODE_ENV = 'test'; - t.context.esAlias = randomId('esalias'); - t.context.esIndex = randomId('esindex'); - process.env.ES_INDEX = t.context.esAlias; - await bootstrapElasticSearch({ - host: 'fakehost', - index: t.context.esIndex, - alias: t.context.esAlias, + process.env.auth_mode = 'private'; + process.env.dbMaxPool = concurrency; + + // Generate a local test postGres database + t.context.testDbName = `granules_${cryptoRandomString({ length: 10 })}`; + const { knex, knexAdmin } = await generateLocalTestDb(t.context.testDbName, migrationDir); + t.context.knex = knex; + t.context.knexAdmin = knexAdmin; + process.env = { + ...process.env, + ...localStackConnectionEnv, + PG_DATABASE: t.context.testDbName, + }; + + const granulePgModel = new GranulePgModel(); + + const collectionName = randomString(5); + const collectionVersion = randomString(3); + const testPgCollection = fakeCollectionRecordFactory({ + name: collectionName, + version: collectionVersion, }); - t.context.esClient = await getEsClient(); - process.env.auth_mode = 'private'; + const collectionPgModel = new CollectionPgModel(); + const collectionPgRecords = await collectionPgModel.create( + knex, + testPgCollection + ); + // iterate 10k times + await pTimes(granuleTotal, ((index) => { + if (index % 1000 === 0 && index !== 0) { + console.log('Creating granule', index); + } + const newPgGranule = fakeGranuleRecordFactory({ + granule_id: randomString(25), + collection_cumulus_id: collectionPgRecords[0].cumulus_id, + }); + return granulePgModel.create(knex, newPgGranule); + }), { concurrency }); }); test.after.always(async (t) => { delete process.env.auth_mode; - await t.context.esClient.client.indices.delete({ index: t.context.esIndex }); + await destroyLocalTestDb({ + knex: t.context.knex, + knexAdmin: t.context.knexAdmin, + testDbName: t.context.testDbName, + }); }); -// TODO postgres query doesn't return searchContext -test.serial.skip('CUMULUS-2930 /GET granules allows searching past 10K results windows with searchContext', async (t) => { - const numGranules = 12 * 1000; - - // create granules in batches of 1000 - for (let i = 0; i < numGranules; i += 1000) { - const granules = granuleFactory(1000); - // eslint-disable-next-line no-await-in-loop - await loadGranules(granules, t); - console.log(`${i} of ${numGranules} loaded`); - } - console.log('Granules loaded.'); - - // expect numGranules / 100 loops since the api limit is 100; - const expectedLoops = 1 + (numGranules / 100); - let actualLoops = 0; - let lastResults = []; - let queryString = ''; - let searchContext = ''; - - do { - actualLoops += 1; - // eslint-disable-next-line no-await-in-loop - const response = await request(app) - .get(`/granules?limit=100${queryString}`) - .set('Accept', 'application/json') - .expect(200); - - const results = response.body.results; - t.notDeepEqual(results, lastResults); - lastResults = results; - - searchContext = response.body.meta.searchContext; - if (searchContext) { - t.is(results.length, 100); - } else { - t.is(results.length, 0); - } - queryString = `&searchContext=${response.body.meta.searchContext}`; - } while (searchContext !== undefined); +test.serial('CUMULUS-2930/3967 /GET granules allows searching past 10K results windows using pagination', async (t) => { + const response = await request(app) + .get('/granules?limit=100&page=101') + .set('Accept', 'application/json') + .expect(200); - t.is(lastResults.length, 0); - t.is(actualLoops, expectedLoops); + t.is(response.body.results.length, 1); }); diff --git a/packages/api/tests/endpoints/providers/create-provider.js b/packages/api/tests/endpoints/providers/create-provider.js index a1596e3e000..b26babaaf3e 100644 --- a/packages/api/tests/endpoints/providers/create-provider.js +++ b/packages/api/tests/endpoints/providers/create-provider.js @@ -20,11 +20,6 @@ const { } = require('@cumulus/aws-client/S3'); const { randomString } = require('@cumulus/common/test-utils'); const { RecordDoesNotExist } = require('@cumulus/errors'); -const { Search } = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const AccessToken = require('../../../models/access-tokens'); const { @@ -33,9 +28,6 @@ const { setAuthorizedOAuthUsers, } = require('../../../lib/testUtils'); const assertions = require('../../../lib/assertions'); -const { post } = require('../../../endpoints/providers'); - -const { buildFakeExpressResponse } = require('../utils'); const testDbName = randomString(12); process.env.AccessTokensTable = randomString(); @@ -69,15 +61,6 @@ test.before(async (t) => { await s3().createBucket({ Bucket: process.env.system_bucket }); - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esProviderClient = new Search( - {}, - 'provider', - t.context.esIndex - ); - const username = randomString(); await setAuthorizedOAuthUsers([username]); @@ -90,7 +73,6 @@ test.before(async (t) => { test.after.always(async (t) => { await recursivelyDeleteS3Bucket(process.env.system_bucket); await accessTokenModel.deleteTable(); - await cleanupTestIndex(t.context); await destroyLocalTestDb({ knex: t.context.testKnex, knexAdmin: t.context.testKnexAdmin, @@ -140,7 +122,7 @@ test('POST with invalid authorization scheme returns an invalid authorization re await providerDoesNotExist(t, newProvider.id); }); -test('POST creates a new provider in all data stores', async (t) => { +test('POST creates a new provider in postgres', async (t) => { const { providerPgModel } = t.context; const newProviderId = randomString(); const newProvider = fakeProviderFactory({ @@ -176,11 +158,6 @@ test('POST creates a new provider in all data stores', async (t) => { postgresOmitList ) ); - - const esRecord = await t.context.esProviderClient.get( - newProvider.id - ); - t.like(esRecord, record); }); test('POST creates a new provider in PG with correct timestamps', async (t) => { @@ -208,15 +185,9 @@ test('POST creates a new provider in PG with correct timestamps', async (t) => { t.true(record.createdAt > newProvider.createdAt); t.true(record.updatedAt > newProvider.updatedAt); - const esRecord = await t.context.esProviderClient.get( - newProvider.id - ); - // PG and ES and returned API records have the same timestamps t.is(providerPgRecord.created_at.getTime(), record.createdAt); t.is(providerPgRecord.updated_at.getTime(), record.updatedAt); - t.is(providerPgRecord.created_at.getTime(), esRecord.createdAt); - t.is(providerPgRecord.updated_at.getTime(), esRecord.updatedAt); }); test('POST returns a 409 error if the provider already exists in postgres', async (t) => { @@ -309,59 +280,3 @@ test('CUMULUS-176 POST returns a 400 response if invalid JSON provided', async ( `response.text: ${response.text}` ); }); - -test('post() does not write to Elasticsearch if writing to PostgreSQL fails', async (t) => { - const provider = fakeProviderFactory(); - - const fakeProviderPgModel = { - create: () => Promise.reject(new Error('something bad')), - exists: () => false, - }; - - const expressRequest = { - body: provider, - testContext: { - providerPgModel: fakeProviderPgModel, - }, - }; - - const response = buildFakeExpressResponse(); - - await post(expressRequest, response); - - t.true(response.boom.badImplementation.calledWithMatch('something bad')); - - t.false(await t.context.esProviderClient.exists( - provider.id - )); -}); - -test('post() does not write to PostgreSQL if writing to Elasticsearch fails', async (t) => { - const provider = fakeProviderFactory(); - - const fakeEsClient = { - initializeEsClient: () => Promise.resolve(), - client: { - index: () => Promise.reject(new Error('something bad')), - }, - }; - - const expressRequest = { - body: provider, - testContext: { - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await post(expressRequest, response); - - t.true(response.boom.badImplementation.calledWithMatch('something bad')); - - t.false( - await t.context.providerPgModel.exists(t.context.testKnex, { - name: provider.id, - }) - ); -}); diff --git a/packages/api/tests/endpoints/providers/delete-provider.js b/packages/api/tests/endpoints/providers/delete-provider.js index 6868ce55ff2..ae3dcdd88a2 100644 --- a/packages/api/tests/endpoints/providers/delete-provider.js +++ b/packages/api/tests/endpoints/providers/delete-provider.js @@ -22,25 +22,14 @@ const { RulePgModel, ProviderPgModel, migrationDir, - translatePostgresProviderToApiProvider, } = require('@cumulus/db'); -const { Search } = require('@cumulus/es-client/search'); -const indexer = require('@cumulus/es-client/indexer'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { AccessToken } = require('../../../models'); const { createFakeJwtAuthToken, setAuthorizedOAuthUsers, - createProviderTestRecords, } = require('../../../lib/testUtils'); const assertions = require('../../../lib/assertions'); -const { del } = require('../../../endpoints/providers'); - -const { buildFakeExpressResponse } = require('../utils'); const testDbName = randomId('db'); @@ -71,15 +60,6 @@ test.before(async (t) => { await s3().createBucket({ Bucket: process.env.system_bucket }); - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esProviderClient = new Search( - {}, - 'provider', - t.context.esIndex - ); - const username = randomId('user'); await setAuthorizedOAuthUsers([username]); @@ -99,19 +79,16 @@ test.before(async (t) => { test.beforeEach(async (t) => { const testPgProvider = fakeProviderRecordFactory(); t.context.testPgProvider = testPgProvider; - const testProvider = translatePostgresProviderToApiProvider(testPgProvider); const [pgProvider] = await t.context.providerPgModel .create( t.context.testKnex, testPgProvider ); t.context.providerCumulusId = pgProvider.cumulus_id; - await indexer.indexProvider(t.context.esClient, testProvider, t.context.esIndex); }); test.after.always(async (t) => { await accessTokenModel.deleteTable(); - await cleanupTestIndex(t.context); await recursivelyDeleteS3Bucket(process.env.system_bucket); await destroyLocalTestDb({ knex: t.context.testKnex, @@ -144,7 +121,7 @@ test('Attempting to delete a provider with an invalid access token returns an un test.todo('Attempting to delete a provider with an unauthorized user returns an unauthorized response'); -test('Deleting a provider removes the provider from all data stores', async (t) => { +test('Deleting a provider removes the provider from postgres', async (t) => { const { testPgProvider, providerPgModel } = t.context; const name = testPgProvider.name; await request(app) @@ -154,70 +131,9 @@ test('Deleting a provider removes the provider from all data stores', async (t) .expect(200); t.false(await providerPgModel.exists(t.context.testKnex, { name })); - t.false( - await t.context.esProviderClient.exists( - testPgProvider.name - ) - ); -}); - -test('Deleting a provider that exists in PostgreSQL and not Elasticsearch succeeds', async (t) => { - const testPgProvider = fakeProviderRecordFactory(); - await t.context.providerPgModel - .create( - t.context.testKnex, - testPgProvider - ); - - await request(app) - .delete(`/providers/${testPgProvider.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.false( - await t.context.providerPgModel.exists( - t.context.testKnex, - { name: testPgProvider.name } - ) - ); - t.false( - await t.context.esProviderClient.exists( - testPgProvider.name - ) - ); -}); - -test('Deleting a provider that exists in Elasticsearch and not PostgreSQL succeeds', async (t) => { - const testPgProvider = fakeProviderRecordFactory(); - const testProvider = translatePostgresProviderToApiProvider(testPgProvider); - await indexer.indexProvider(t.context.esClient, testProvider, t.context.esIndex); - - t.true( - await t.context.esProviderClient.exists( - testPgProvider.name - ) - ); - - await request(app) - .delete(`/providers/${testPgProvider.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - t.false( - await t.context.providerPgModel.exists( - t.context.testKnex, - { name: testPgProvider.name } - ) - ); - t.false( - await t.context.esProviderClient.exists( - testPgProvider.name - ) - ); }); -test('Deleting a provider that does not exist in PostgreSQL and Elasticsearch returns a 404', async (t) => { +test('Deleting a provider that does not exist in PostgreSQL returns a 404', async (t) => { const { status } = await request(app) .delete(`/providers/${randomString}`) .set('Accept', 'application/json') @@ -247,95 +163,6 @@ test('Attempting to delete a provider with an associated postgres rule returns a t.true(response.body.message.includes('Cannot delete provider with associated rules')); }); -test('del() does not remove from Elasticsearch if removing from PostgreSQL fails', async (t) => { - const { - originalPgRecord, - } = await createProviderTestRecords( - t.context - ); - - const fakeproviderPgModel = { - delete: () => { - throw new Error('something bad'); - }, - get: () => Promise.resolve(originalPgRecord), - }; - - const expressRequest = { - params: { - id: originalPgRecord.id, - }, - testContext: { - knex: t.context.testKnex, - providerPgModel: fakeproviderPgModel, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - del(expressRequest, response), - { message: 'something bad' } - ); - - t.true( - await t.context.providerPgModel.exists(t.context.testKnex, { - name: originalPgRecord.name, - }) - ); - t.true( - await t.context.esProviderClient.exists( - originalPgRecord.name - ) - ); -}); - -test('del() does not remove from PostgreSQL if removing from Elasticsearch fails', async (t) => { - const { - originalProvider, - } = await createProviderTestRecords( - t.context - ); - - const fakeEsClient = { - initializeEsClient: () => Promise.resolve(), - client: { - delete: () => { - throw new Error('something bad'); - }, - }, - }; - - const expressRequest = { - params: { - id: originalProvider.id, - }, - body: originalProvider, - testContext: { - knex: t.context.testKnex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - del(expressRequest, response), - { message: 'something bad' } - ); - - t.true( - await t.context.providerPgModel.exists(t.context.testKnex, { - name: originalProvider.id, - }) - ); - t.true( - await t.context.esProviderClient.exists( - originalProvider.id - ) - ); -}); - test('Attempting to delete a provider with an associated granule does not delete the provider', async (t) => { const { collectionPgModel, diff --git a/packages/api/tests/endpoints/providers/get-provider.js b/packages/api/tests/endpoints/providers/get-provider.js index 47f746be540..87ccba33fc8 100644 --- a/packages/api/tests/endpoints/providers/get-provider.js +++ b/packages/api/tests/endpoints/providers/get-provider.js @@ -9,8 +9,6 @@ const { recursivelyDeleteS3Bucket, } = require('@cumulus/aws-client/S3'); const { randomString } = require('@cumulus/common/test-utils'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const { getEsClient } = require('@cumulus/es-client/search'); const { destroyLocalTestDb, fakeProviderRecordFactory, @@ -32,9 +30,6 @@ process.env.stackName = randomString(); process.env.system_bucket = randomString(); process.env.TOKEN_SECRET = randomString(); -const esIndex = randomString(); -let esClient; - let jwtAuthToken; let accessTokenModel; @@ -42,14 +37,6 @@ test.before(async (t) => { t.context.testDbName = `test_executions_${cryptoRandomString({ length: 10 })}`; await s3().createBucket({ Bucket: process.env.system_bucket }); - const esAlias = randomString(); - process.env.ES_INDEX = esAlias; - await bootstrapElasticSearch({ - host: 'fakehost', - index: esIndex, - alias: esAlias, - }); - const username = randomString(); await setAuthorizedOAuthUsers([username]); @@ -59,7 +46,6 @@ test.before(async (t) => { jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); - esClient = await getEsClient('fakehost'); const { knex, knexAdmin } = await generateLocalTestDb(t.context.testDbName, migrationDir); t.context.knex = knex; t.context.knexAdmin = knexAdmin; @@ -83,7 +69,6 @@ test.beforeEach(async (t) => { test.after.always(async (t) => { await recursivelyDeleteS3Bucket(process.env.system_bucket); await accessTokenModel.deleteTable(); - await esClient.client.indices.delete({ index: esIndex }); await destroyLocalTestDb({ knex: t.context.knex, knexAdmin: t.context.knexAdmin, diff --git a/packages/api/tests/endpoints/providers/list-providers.js b/packages/api/tests/endpoints/providers/list-providers.js index 40c33d4ddd9..fd7f3b945fa 100644 --- a/packages/api/tests/endpoints/providers/list-providers.js +++ b/packages/api/tests/endpoints/providers/list-providers.js @@ -8,9 +8,6 @@ const { recursivelyDeleteS3Bucket, } = require('@cumulus/aws-client/S3'); const { randomString } = require('@cumulus/common/test-utils'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const { getEsClient } = require('@cumulus/es-client/search'); -const indexer = require('@cumulus/es-client/indexer'); const { ProviderPgModel, @@ -35,9 +32,6 @@ process.env.TOKEN_SECRET = randomString(); // import the express app after setting the env variables const { app } = require('../../../app'); -const esIndex = randomString(); -let esClient; - let jwtAuthToken; let accessTokenModel; @@ -48,18 +42,7 @@ test.before(async (t) => { const username = randomString(); await setAuthorizedOAuthUsers([username]); - - const esAlias = randomString(); - process.env.ES_INDEX = esAlias; - - await Promise.all([ - accessTokenModel.createTable(), - bootstrapElasticSearch({ - host: 'fakehost', - index: esIndex, - alias: esAlias, - }), - ]); + await accessTokenModel.createTable(); t.context.testDbName = `test_providers_${cryptoRandomString({ length: 10 })}`; @@ -74,13 +57,17 @@ test.before(async (t) => { jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); - esClient = await getEsClient('fakehost'); + t.context.testProvider = fakeProviderRecordFactory(); + t.context.providerPgModel = new ProviderPgModel(); + await t.context.providerPgModel.insert( + t.context.knex, + t.context.testProvider + ); }); test.after.always((t) => Promise.all([ recursivelyDeleteS3Bucket(process.env.system_bucket), accessTokenModel.deleteTable(), - esClient.client.indices.delete({ index: esIndex }), destroyLocalTestDb({ ...t.context, }), @@ -108,12 +95,6 @@ test('CUMULUS-912 GET without pathParameters and with an invalid access token re test.todo('CUMULUS-912 GET without pathParameters and with an unauthorized user returns an unauthorized response'); test('default returns list of providers', async (t) => { - const testProvider = fakeProviderRecordFactory(); - const providerPgModel = new ProviderPgModel(); - const [provider] = await providerPgModel.create(t.context.knex, testProvider); - const pgProvider = await providerPgModel.get(t.context.knex, { cumulus_id: provider.cumulus_id }); - await indexer.indexProvider(esClient, pgProvider, esIndex); - const response = await request(app) .get('/providers') .set('Accept', 'application/json') @@ -121,5 +102,5 @@ test('default returns list of providers', async (t) => { .expect(200); const { results } = response.body; - t.truthy(results.find((r) => r.id === testProvider.id)); + t.truthy(results.find((r) => r.id === t.context.testProvider.name)); }); diff --git a/packages/api/tests/endpoints/providers/update-provider.js b/packages/api/tests/endpoints/providers/update-provider.js index 81e0df725a2..637a927403f 100644 --- a/packages/api/tests/endpoints/providers/update-provider.js +++ b/packages/api/tests/endpoints/providers/update-provider.js @@ -16,26 +16,16 @@ const { translateApiProviderToPostgresProvider, ProviderPgModel, migrationDir, - fakeProviderRecordFactory, - translatePostgresProviderToApiProvider, } = require('@cumulus/db'); -const { Search } = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { AccessToken } = require('../../../models'); const { createFakeJwtAuthToken, fakeProviderFactory, setAuthorizedOAuthUsers, - createProviderTestRecords, } = require('../../../lib/testUtils'); const assertions = require('../../../lib/assertions'); -const { put } = require('../../../endpoints/providers'); -const { buildFakeExpressResponse } = require('../utils'); const testDbName = randomString(12); @@ -62,15 +52,6 @@ test.before(async (t) => { await s3().createBucket({ Bucket: process.env.system_bucket }); - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esProviderClient = new Search( - {}, - 'provider', - t.context.esIndex - ); - const username = randomString(); await setAuthorizedOAuthUsers([username]); @@ -95,7 +76,6 @@ test.beforeEach(async (t) => { test.after.always(async (t) => { await recursivelyDeleteS3Bucket(process.env.system_bucket); await accessTokenModel.deleteTable(); - await cleanupTestIndex(t.context); await destroyLocalTestDb({ knex: t.context.testKnex, knexAdmin: t.context.testKnexAdmin, @@ -160,18 +140,6 @@ test('PUT updates existing provider', async (t) => { postgresOmitList ) ); - - const updatedEsRecord = await t.context.esProviderClient.get( - testProvider.id - ); - t.like( - updatedEsRecord, - { - ...expectedProvider, - updatedAt: actualPostgresProvider.updated_at.getTime(), - timestamp: updatedEsRecord.timestamp, - } - ); }); test('PUT updates existing provider and correctly removes fields', async (t) => { @@ -216,7 +184,7 @@ test('PUT updates existing provider and correctly removes fields', async (t) => t.is(actualPostgresProvider.global_connection_limit, null); }); -test('PUT updates existing provider in all data stores with correct timestamps', async (t) => { +test('PUT updates existing provider in postgres with correct timestamps', async (t) => { const { testProvider, testProvider: { id } } = t.context; const expectedProvider = omit(testProvider, ['globalConnectionLimit', 'protocol', 'cmKeyId']); @@ -238,16 +206,10 @@ test('PUT updates existing provider in all data stores with correct timestamps', t.context.testKnex, { name: id } ); - const updatedEsRecord = await t.context.esProviderClient.get( - testProvider.id - ); t.true(actualPostgresProvider.updated_at.getTime() > updatedProvider.updatedAt); // createdAt timestamp from original record should have been preserved t.is(actualPostgresProvider.created_at.getTime(), testProvider.createdAt); - // PG and ES records have the same timestamps - t.is(actualPostgresProvider.created_at.getTime(), updatedEsRecord.createdAt); - t.is(actualPostgresProvider.updated_at.getTime(), updatedEsRecord.updatedAt); }); test('PUT returns 404 for non-existent provider', async (t) => { @@ -309,114 +271,3 @@ test('PUT without an Authorization header returns an Authorization Missing respo ); t.is(provider.name, t.context.testPostgresProvider.name); }); - -test('put() does not write to Elasticsearch if writing to PostgreSQL fails', async (t) => { - const { testKnex } = t.context; - const { - originalProvider, - originalPgRecord, - originalEsRecord, - } = await createProviderTestRecords( - t.context, - { - host: 'first-host', - } - ); - - const fakeproviderPgModel = { - upsert: () => Promise.reject(new Error('something bad')), - get: () => fakeProviderRecordFactory({ created_at: new Date() }), - }; - - const updatedProvider = { - ...originalProvider, - host: 'second-host', - }; - - const expressRequest = { - params: { - id: updatedProvider.id, - }, - body: updatedProvider, - testContext: { - knex: testKnex, - providerPgModel: fakeproviderPgModel, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - put(expressRequest, response), - { message: 'something bad' } - ); - - t.deepEqual( - await t.context.providerPgModel.get(t.context.testKnex, { - name: updatedProvider.id, - }), - originalPgRecord - ); - t.deepEqual( - await t.context.esProviderClient.get( - originalProvider.id - ), - originalEsRecord - ); -}); - -test('put() does not write to PostgreSQL if writing to Elasticsearch fails', async (t) => { - const { testKnex } = t.context; - const { - originalPgRecord, - originalEsRecord, - } = await createProviderTestRecords( - t.context, - { - host: 'first-host', - } - ); - - const fakeEsClient = { - initializeEsClient: () => Promise.resolve(), - client: { - index: () => Promise.reject(new Error('something bad')), - }, - }; - const apiProvider = translatePostgresProviderToApiProvider(originalPgRecord); - const updatedProvider = { - ...apiProvider, - host: 'second-host', - }; - - const expressRequest = { - params: { - id: updatedProvider.id, - }, - body: updatedProvider, - testContext: { - knex: testKnex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - put(expressRequest, response), - { message: 'something bad' } - ); - - t.deepEqual( - await t.context.providerPgModel.get(t.context.testKnex, { - name: updatedProvider.id, - }), - originalPgRecord - ); - t.deepEqual( - await t.context.esProviderClient.get( - originalPgRecord.name - ), - originalEsRecord - ); -}); diff --git a/packages/api/tests/endpoints/test-elasticsearch.js b/packages/api/tests/endpoints/test-elasticsearch.js deleted file mode 100644 index 811587e93c3..00000000000 --- a/packages/api/tests/endpoints/test-elasticsearch.js +++ /dev/null @@ -1,698 +0,0 @@ -'use strict'; - -const request = require('supertest'); -const test = require('ava'); -const get = require('lodash/get'); -const sinon = require('sinon'); - -const { - localStackConnectionEnv, - generateLocalTestDb, - destroyLocalTestDb, - migrationDir, -} = require('@cumulus/db'); -const awsServices = require('@cumulus/aws-client/services'); -const { - recursivelyDeleteS3Bucket, -} = require('@cumulus/aws-client/S3'); -const { randomString, randomId } = require('@cumulus/common/test-utils'); -const { IndexExistsError } = require('@cumulus/errors'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const { getEsClient, defaultIndexAlias } = require('@cumulus/es-client/search'); -const mappings = require('@cumulus/es-client/config/mappings.json'); -const startAsyncOperation = require('../../lib/startAsyncOperation'); - -const models = require('../../models'); -const assertions = require('../../lib/assertions'); -const { - createFakeJwtAuthToken, - setAuthorizedOAuthUsers, -} = require('../../lib/testUtils'); - -const esIndex = randomId('esindex'); - -process.env.AccessTokensTable = randomString(); -process.env.TOKEN_SECRET = randomString(); -process.env.stackName = randomString(); -process.env.system_bucket = randomString(); - -// import the express app after setting the env variables -const { app } = require('../../app'); -const { indexFromDatabase } = require('../../endpoints/elasticsearch'); - -let jwtAuthToken; -let accessTokenModel; -let esClient; - -/** - * Index fake data - * - * @returns {undefined} - none - */ -async function indexData() { - const rules = [ - { name: 'Rule1' }, - { name: 'Rule2' }, - { name: 'Rule3' }, - ]; - - await Promise.all(rules.map(async (rule) => { - await esClient.client.index({ - index: esIndex, - type: 'rule', - id: rule.name, - body: rule, - }); - })); - - await esClient.client.indices.refresh(); -} - -/** - * Create and alias index by going through ES bootstrap - * - * @param {string} indexName - index name - * @param {string} aliasName - alias name - * @returns {undefined} - none - */ -async function createIndex(indexName, aliasName) { - await bootstrapElasticSearch({ - host: 'fakehost', - index: indexName, - alias: aliasName, - }); - esClient = await getEsClient(); -} - -const testDbName = randomId('elasticsearch'); - -test.before(async (t) => { - await awsServices.s3().createBucket({ Bucket: process.env.system_bucket }); - - const username = randomString(); - await setAuthorizedOAuthUsers([username]); - - accessTokenModel = new models.AccessToken(); - await accessTokenModel.createTable(); - - jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); - - t.context.esAlias = randomString(); - process.env.ES_INDEX = t.context.esAlias; - process.env = { - ...process.env, - ...localStackConnectionEnv, - PG_DATABASE: testDbName, - }; - - const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); - t.context.testKnex = knex; - t.context.testKnexAdmin = knexAdmin; - - // create the elasticsearch index and add mapping - await createIndex(esIndex, t.context.esAlias); - - await indexData(); -}); - -test.after.always(async (t) => { - await accessTokenModel.deleteTable(); - await esClient.client.indices.delete({ index: esIndex }); - await destroyLocalTestDb({ - knex: t.context.testKnex, - knexAdmin: t.context.testKnexAdmin, - testDbName, - }); - await recursivelyDeleteS3Bucket(process.env.system_bucket); -}); - -test('PUT snapshot without an Authorization header returns an Authorization Missing response', async (t) => { - const response = await request(app) - .post('/elasticsearch/create-snapshot') - .set('Accept', 'application/json') - .expect(401); - - assertions.isAuthorizationMissingResponse(t, response); -}); - -test('PUT snapshot with an invalid access token returns an unauthorized response', async (t) => { - const response = await request(app) - .post('/elasticsearch/create-snapshot') - .set('Accept', 'application/json') - .set('Authorization', 'Bearer ThisIsAnInvalidAuthorizationToken') - .expect(401); - - assertions.isInvalidAccessTokenResponse(t, response); -}); - -test.serial('Reindex - multiple aliases found', async (t) => { - // Prefixes for error message predictability - const indexName = `z-${randomString()}`; - const otherIndexName = `a-${randomString()}`; - - const aliasName = randomString(); - - await esClient.client.indices.create({ - index: indexName, - body: { mappings }, - }); - - await esClient.client.indices.putAlias({ - index: indexName, - name: aliasName, - }); - - await esClient.client.indices.create({ - index: otherIndexName, - body: { mappings }, - }); - - await esClient.client.indices.putAlias({ - index: otherIndexName, - name: aliasName, - }); - - const response = await request(app) - .post('/elasticsearch/reindex') - .send({ aliasName }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, `Multiple indices found for alias ${aliasName}. Specify source index as one of [${otherIndexName}, ${indexName}].`); - - await esClient.client.indices.delete({ index: indexName }); - await esClient.client.indices.delete({ index: otherIndexName }); -}); - -test.serial('Reindex - specify a source index that does not exist', async (t) => { - const { esAlias } = t.context; - - const response = await request(app) - .post('/elasticsearch/reindex') - .send({ aliasName: esAlias, sourceIndex: 'source-index' }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, 'Source index source-index does not exist.'); -}); - -test.serial('Reindex - specify a source index that is not aliased', async (t) => { - const { esAlias } = t.context; - const indexName = 'source-index'; - const destIndex = randomString(); - - await esClient.client.indices.create({ - index: indexName, - body: { mappings }, - }); - - const response = await request(app) - .post('/elasticsearch/reindex') - .send({ - aliasName: esAlias, - sourceIndex: indexName, - destIndex, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.is(response.body.message, `Reindexing to ${destIndex} from ${indexName}. Check the reindex-status endpoint for status.`); - - // Check the reindex status endpoint to see if the operation has completed - let statusResponse = await request(app) - .get('/elasticsearch/reindex-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - /* eslint-disable no-await-in-loop */ - while (Object.keys(statusResponse.body.reindexStatus.nodes).length > 0) { - statusResponse = await request(app) - .get('/elasticsearch/reindex-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - } - /* eslint-enable no-await-in-loop */ - - await esClient.client.indices.delete({ index: indexName }); - await esClient.client.indices.delete({ index: destIndex }); -}); - -test.serial('Reindex request returns 400 with the expected message when source index matches destination index.', async (t) => { - const indexName = randomId('index'); - await esClient.client.indices.create({ - index: indexName, - body: { mappings }, - }); - - const response = await request(app) - .post('/elasticsearch/reindex') - .send({ destIndex: indexName, sourceIndex: indexName }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, `source index(${indexName}) and destination index(${indexName}) must be different.`); - await esClient.client.indices.delete({ index: indexName }); -}); - -test.serial('Reindex request returns 400 with the expected message when source index matches the default destination index.', async (t) => { - const date = new Date(); - const defaultIndexName = `cumulus-${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}`; - - try { - await createIndex(defaultIndexName); - } catch (error) { - if (!(error instanceof IndexExistsError)) throw error; - } - - t.teardown(async () => { - await esClient.client.indices.delete({ index: defaultIndexName }); - }); - - const response = await request(app) - .post('/elasticsearch/reindex') - .send({ sourceIndex: defaultIndexName }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, `source index(${defaultIndexName}) and destination index(${defaultIndexName}) must be different.`); -}); - -test.serial('Reindex success', async (t) => { - const { esAlias } = t.context; - const destIndex = randomString(); - - const response = await request(app) - .post('/elasticsearch/reindex') - .send({ - aliasName: esAlias, - destIndex, - sourceIndex: esIndex, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.is(response.body.message, `Reindexing to ${destIndex} from ${esIndex}. Check the reindex-status endpoint for status.`); - - // Check the reindex status endpoint to see if the operation has completed - let statusResponse = await request(app) - .get('/elasticsearch/reindex-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - /* eslint-disable no-await-in-loop */ - while (Object.keys(statusResponse.body.reindexStatus.nodes).length > 0) { - statusResponse = await request(app) - .get('/elasticsearch/reindex-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - } - /* eslint-enable no-await-in-loop */ - - const indexStatus = statusResponse.body.indexStatus.indices[destIndex]; - - t.is(3, indexStatus.primaries.docs.count); - - // Validate destination index mappings are correct - const fieldMappings = await esClient.client.indices.getMapping() - .then((mappingsResponse) => mappingsResponse.body); - - const sourceMapping = get(fieldMappings, esIndex); - const destMapping = get(fieldMappings, destIndex); - - t.deepEqual(sourceMapping.mappings, destMapping.mappings); - - await esClient.client.indices.delete({ index: destIndex }); -}); - -test.serial('Reindex - destination index exists', async (t) => { - const { esAlias } = t.context; - const destIndex = randomString(); - const newAlias = randomString(); - - await createIndex(destIndex, newAlias); - - const response = await request(app) - .post('/elasticsearch/reindex') - .send({ - aliasName: esAlias, - destIndex: destIndex, - sourceIndex: esIndex, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.is(response.body.message, `Reindexing to ${destIndex} from ${esIndex}. Check the reindex-status endpoint for status.`); - - // Check the reindex status endpoint to see if the operation has completed - let statusResponse = await request(app) - .get('/elasticsearch/reindex-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - /* eslint-disable no-await-in-loop */ - while (Object.keys(statusResponse.body.reindexStatus.nodes).length > 0) { - statusResponse = await request(app) - .get('/elasticsearch/reindex-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - } - /* eslint-enable no-await-in-loop */ - - await esClient.client.indices.delete({ index: destIndex }); -}); - -test.serial('Reindex status, no task running', async (t) => { - const response = await request(app) - .get('/elasticsearch/reindex-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.deepEqual(response.body.reindexStatus, { nodes: {} }); -}); - -test.serial('Change index - no current', async (t) => { - const { esAlias } = t.context; - - const response = await request(app) - .post('/elasticsearch/change-index') - .send({ - aliasName: esAlias, - newIndex: 'dest-index', - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, 'Please explicity specify a current and new index.'); -}); - -test.serial('Change index - no new', async (t) => { - const { esAlias } = t.context; - - const response = await request(app) - .post('/elasticsearch/change-index') - .send({ - aliasName: esAlias, - currentIndex: 'source-index', - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, 'Please explicity specify a current and new index.'); -}); - -test.serial('Change index - current index does not exist', async (t) => { - const { esAlias } = t.context; - - const currentIndex = 'source-index'; - - const response = await request(app) - .post('/elasticsearch/change-index') - .send({ - aliasName: esAlias, - currentIndex, - newIndex: 'dest-index', - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, `Current index ${currentIndex} does not exist.`); -}); - -test.serial('Change index - new index does not exist', async (t) => { - const { esAlias } = t.context; - - const newIndex = 'dest-index'; - - const response = await request(app) - .post('/elasticsearch/change-index') - .send({ - aliasName: esAlias, - currentIndex: esIndex, - newIndex, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.is(response.body.message, `Change index success - alias ${esAlias} now pointing to ${newIndex}`); - - await esClient.client.indices.delete({ index: newIndex }); -}); - -test.serial('Change index - current index same as new index', async (t) => { - const { esAlias } = t.context; - - const response = await request(app) - .post('/elasticsearch/change-index') - .send({ - aliasName: esAlias, - currentIndex: 'source', - newIndex: 'source', - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(400); - - t.is(response.body.message, 'The current index cannot be the same as the new index.'); -}); - -test.serial('Change index', async (t) => { - const sourceIndex = randomString(); - const aliasName = randomString(); - const destIndex = randomString(); - - await createIndex(sourceIndex, aliasName); - - await request(app) - .post('/elasticsearch/reindex') - .send({ - aliasName, - sourceIndex, - destIndex, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const response = await request(app) - .post('/elasticsearch/change-index') - .send({ - aliasName, - currentIndex: sourceIndex, - newIndex: destIndex, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.is(response.body.message, - `Change index success - alias ${aliasName} now pointing to ${destIndex}`); - - const alias = await esClient.client.indices.getAlias({ name: aliasName }) - .then((aliasResponse) => aliasResponse.body); - - // Test that the only index connected to the alias is the destination index - t.deepEqual(Object.keys(alias), [destIndex]); - - t.is((await esClient.client.indices.exists({ index: sourceIndex })).body, true); - - await esClient.client.indices.delete({ index: destIndex }); -}); - -test.serial('Change index and delete source index', async (t) => { - const sourceIndex = randomString(); - const aliasName = randomString(); - const destIndex = randomString(); - - await createIndex(sourceIndex, aliasName); - - await request(app) - .post('/elasticsearch/reindex') - .send({ - aliasName, - sourceIndex, - destIndex, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const response = await request(app) - .post('/elasticsearch/change-index') - .send({ - aliasName, - currentIndex: sourceIndex, - newIndex: destIndex, - deleteSource: true, - }) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.is(response.body.message, - `Change index success - alias ${aliasName} now pointing to ${destIndex} and index ${sourceIndex} deleted`); - t.is((await esClient.client.indices.exists({ index: sourceIndex })).body, false); - - await esClient.client.indices.delete({ index: destIndex }); -}); - -test.serial('Reindex from database - startAsyncOperation is called with expected payload', async (t) => { - const indexName = randomString(); - const processEnv = { ...process.env }; - process.env.ES_HOST = 'fakeEsHost'; - process.env.ReconciliationReportsTable = 'fakeReportsTable'; - - const asyncOperationsStub = sinon.stub(startAsyncOperation, 'invokeStartAsyncOperationLambda'); - const payload = { - indexName, - esRequestConcurrency: 'fakeEsRequestConcurrency', - postgresResultPageSize: 'fakePostgresResultPageSize', - postgresConnectionPoolSize: 'fakePostgresConnectionPoolSize', - }; - - try { - await request(app) - .post('/elasticsearch/index-from-database') - .send( - payload - ) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - t.deepEqual(asyncOperationsStub.getCall(0).args[0].payload, { - ...payload, - esHost: process.env.ES_HOST, - reconciliationReportsTable: process.env.ReconciliationReportsTable, - }); - } finally { - process.env = processEnv; - await esClient.client.indices.delete({ index: indexName }); - asyncOperationsStub.restore(); - } -}); - -test.serial('Indices status', async (t) => { - const indexName = `z-${randomString()}`; - const otherIndexName = `a-${randomString()}`; - - await esClient.client.indices.create({ - index: indexName, - body: { mappings }, - }); - - await esClient.client.indices.create({ - index: otherIndexName, - body: { mappings }, - }); - - const response = await request(app) - .get('/elasticsearch/indices-status') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.true(response.text.includes(indexName)); - t.true(response.text.includes(otherIndexName)); - - await esClient.client.indices.delete({ index: indexName }); - await esClient.client.indices.delete({ index: otherIndexName }); -}); - -test.serial('Current index - default alias', async (t) => { - const indexName = randomString(); - await createIndex(indexName, defaultIndexAlias); - t.teardown(() => esClient.client.indices.delete({ index: indexName })); - - const response = await request(app) - .get('/elasticsearch/current-index') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.true(response.body.includes(indexName)); -}); - -test.serial('Current index - custom alias', async (t) => { - const indexName = randomString(); - const customAlias = randomString(); - await createIndex(indexName, customAlias); - - const response = await request(app) - .get(`/elasticsearch/current-index/${customAlias}`) - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - t.deepEqual(response.body, [indexName]); - - await esClient.client.indices.delete({ index: indexName }); -}); - -test.serial('request to /elasticsearch/index-from-database endpoint returns 500 if invoking StartAsyncOperation lambda throws unexpected error', async (t) => { - const asyncOperationStartStub = sinon.stub(startAsyncOperation, 'invokeStartAsyncOperationLambda').throws( - new Error('failed to start') - ); - - try { - const response = await request(app) - .post('/elasticsearch/index-from-database') - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send({}); - t.is(response.status, 500); - } finally { - asyncOperationStartStub.restore(); - } -}); - -test.serial('indexFromDatabase request completes successfully', async (t) => { - const stub = sinon.stub(startAsyncOperation, 'invokeStartAsyncOperationLambda'); - const functionName = randomId('lambda'); - const fakeRequest = { - apiGateway: { - context: { - functionName, - }, - }, - body: { - indexName: t.context.esAlias, - }, - }; - - const fakeResponse = { - send: sinon.stub(), - }; - - await t.notThrowsAsync(indexFromDatabase(fakeRequest, fakeResponse)); - t.true(fakeResponse.send.called); - stub.restore(); -}); - -test.serial('indexFromDatabase uses correct caller lambda function name', async (t) => { - const stub = sinon.stub(startAsyncOperation, 'invokeStartAsyncOperationLambda'); - const functionName = randomId('lambda'); - const fakeRequest = { - apiGateway: { - context: { - functionName, - }, - }, - body: { - indexName: randomId('index'), - }, - }; - const fakeResponse = { - send: sinon.stub(), - }; - - await indexFromDatabase(fakeRequest, fakeResponse); - t.is(stub.getCall(0).firstArg.callerLambdaName, functionName); - stub.restore(); -}); diff --git a/packages/api/tests/endpoints/test-executions.js b/packages/api/tests/endpoints/test-executions.js index a192c35d3a1..fac59d3f4d5 100644 --- a/packages/api/tests/endpoints/test-executions.js +++ b/packages/api/tests/endpoints/test-executions.js @@ -78,8 +78,6 @@ process.env.TOKEN_SECRET = randomId('secret'); test.before(async (t) => { process.env = { ...process.env, - ...localStackConnectionEnv, - PG_DATABASE: testDbName, METRICS_ES_HOST: 'fakehost', METRICS_ES_USER: randomId('metricsUser'), METRICS_ES_PASS: randomId('metricsPass'), @@ -235,7 +233,7 @@ test.beforeEach(async (t) => { ]; // create fake Postgres granule records - // es records are for Metrics search + // es records are for Cloud Metrics search t.context.fakePGGranules = await Promise.all(t.context.fakeGranules.map(async (fakeGranule) => { await indexer.indexGranule(esClient, fakeGranule, esIndex); const granulePgRecord = await translateApiGranuleToPostgresGranule({ diff --git a/packages/api/tests/endpoints/test-granules-get.js b/packages/api/tests/endpoints/test-granules-get.js index 493bf89fd69..adc725cb4da 100644 --- a/packages/api/tests/endpoints/test-granules-get.js +++ b/packages/api/tests/endpoints/test-granules-get.js @@ -293,52 +293,6 @@ test.afterEach(async (t) => { }); }); -// TODO postgres query doesn't return searchContext -test.serial.skip('default lists and paginates correctly with search_after', async (t) => { - const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); - const response = await request(app) - .get('/granules') - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const { meta, results } = response.body; - t.is(results.length, 3); - t.is(meta.stack, process.env.stackName); - t.is(meta.table, 'granule'); - t.is(meta.count, 3); - results.forEach((r) => { - t.true(granuleIds.includes(r.granuleId)); - }); - // default paginates correctly with search_after - const firstResponse = await request(app) - .get('/granules?limit=1') - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const { meta: firstMeta, results: firstResults } = firstResponse.body; - t.is(firstResults.length, 1); - t.is(firstMeta.page, 1); - t.truthy(firstMeta.searchContext); - - const newResponse = await request(app) - .get(`/granules?limit=1&page=2&searchContext=${firstMeta.searchContext}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const { meta: newMeta, results: newResults } = newResponse.body; - t.is(newResults.length, 1); - t.is(newMeta.page, 2); - t.truthy(newMeta.searchContext); - - t.true(granuleIds.includes(results[0].granuleId)); - t.true(granuleIds.includes(newResults[0].granuleId)); - t.not(results[0].granuleId, newResults[0].granuleId); - t.not(meta.searchContext === newMeta.searchContext); -}); - test.serial('default lists and paginates correctly from querying database', async (t) => { const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); const response = await request(app) @@ -543,39 +497,7 @@ test.serial('GET returns a 404 response if the granule is not found', async (t) t.is(message, 'Granule not found'); }); -// TODO postgres query doesn't return searchContext -test.serial.skip('default paginates correctly with search_after', async (t) => { - const response = await request(app) - .get('/granules?limit=1') - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); - - const { meta, results } = response.body; - t.is(results.length, 1); - t.is(meta.page, 1); - t.truthy(meta.searchContext); - - const newResponse = await request(app) - .get(`/granules?limit=1&page=2&searchContext=${meta.searchContext}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const { meta: newMeta, results: newResults } = newResponse.body; - t.is(newResults.length, 1); - t.is(newMeta.page, 2); - t.truthy(newMeta.searchContext); - console.log(`default paginates granuleIds: ${JSON.stringify(granuleIds)}, results: ${results[0].granuleId}, ${newResults[0].granuleId}`); - t.true(granuleIds.includes(results[0].granuleId)); - t.true(granuleIds.includes(newResults[0].granuleId)); - t.not(results[0].granuleId, newResults[0].granuleId); - t.not(meta.searchContext === newMeta.searchContext); -}); - -test.only('LIST endpoint returns search result correctly', async (t) => { +test.serial('LIST endpoint returns search result correctly', async (t) => { const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); const searchParams = new URLSearchParams({ granuleId: granuleIds[3], diff --git a/packages/api/tests/endpoints/test-granules.js b/packages/api/tests/endpoints/test-granules.js index 845e395d33b..2d242e17e07 100644 --- a/packages/api/tests/endpoints/test-granules.js +++ b/packages/api/tests/endpoints/test-granules.js @@ -3352,38 +3352,6 @@ test.serial('PUT returns 404 if collection is not part of URI', async (t) => { t.is(response.statusCode, 404); }); -// TODO postgres query doesn't return searchContext -test.serial.skip('default paginates correctly with search_after', async (t) => { - const response = await request(app) - .get('/granules?limit=1') - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const granuleIds = t.context.fakePGGranules.map((i) => i.granule_id); - - const { meta, results } = response.body; - t.is(results.length, 1); - t.is(meta.page, 1); - t.truthy(meta.searchContext); - - const newResponse = await request(app) - .get(`/granules?limit=1&page=2&searchContext=${meta.searchContext}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - - const { meta: newMeta, results: newResults } = newResponse.body; - t.is(newResults.length, 1); - t.is(newMeta.page, 2); - t.truthy(newMeta.searchContext); - console.log(`default paginates granuleIds: ${JSON.stringify(granuleIds)}, results: ${results[0].granuleId}, ${newResults[0].granuleId}`); - t.true(granuleIds.includes(results[0].granuleId)); - t.true(granuleIds.includes(newResults[0].granuleId)); - t.not(results[0].granuleId, newResults[0].granuleId); - t.not(meta.searchContext === newMeta.searchContext); -}); - test.serial('PUT returns 400 for version value less than the configured value', async (t) => { const granuleId = t.context.createGranuleId(); const response = await request(app) diff --git a/packages/api/tests/endpoints/test-pdrs.js b/packages/api/tests/endpoints/test-pdrs.js index 562cf33598e..65abd558693 100644 --- a/packages/api/tests/endpoints/test-pdrs.js +++ b/packages/api/tests/endpoints/test-pdrs.js @@ -3,6 +3,7 @@ const test = require('ava'); const request = require('supertest'); const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); const awsServices = require('@cumulus/aws-client/services'); const { recursivelyDeleteS3Bucket, @@ -19,7 +20,6 @@ const { migrationDir, PdrPgModel, ProviderPgModel, - translatePostgresPdrToApiPdr, } = require('@cumulus/db'); const { fakeCollectionRecordFactory, @@ -27,12 +27,6 @@ const { fakePdrRecordFactory, fakeProviderRecordFactory, } = require('@cumulus/db/dist/test-utils'); -const indexer = require('@cumulus/es-client/indexer'); -const { Search } = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { constructCollectionId } = require('@cumulus/message/Collections'); const { @@ -40,7 +34,6 @@ const { fakePdrFactory, setAuthorizedOAuthUsers, createPdrTestRecords, - fakePdrFactoryV2, } = require('../../lib/testUtils'); const models = require('../../models'); const assertions = require('../../lib/assertions'); @@ -59,7 +52,6 @@ const pdrS3Key = (pdrName) => `${process.env.stackName}/pdrs/${pdrName}`; // create all the variables needed across this test const testDbName = `pdrs_${cryptoRandomString({ length: 10 })}`; -let fakePdrs; let jwtAuthToken; let accessTokenModel; @@ -76,15 +68,6 @@ test.before(async (t) => { t.context.knex = knex; t.context.knexAdmin = knexAdmin; - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esPdrsClient = new Search( - {}, - 'pdr', - t.context.esIndex - ); - // create a fake bucket await awsServices.s3().createBucket({ Bucket: process.env.system_bucket }); @@ -98,14 +81,6 @@ test.before(async (t) => { jwtAuthToken = await createFakeJwtAuthToken({ accessTokenModel, username }); - // create fake PDR records - fakePdrs = ['completed', 'failed'].map(fakePdrFactory); - await Promise.all( - fakePdrs.map( - (pdr) => indexer.indexPdr(t.context.esClient, pdr, t.context.esIndex) - ) - ); - // Create a PG Collection t.context.testPgCollection = fakeCollectionRecordFactory(); const collectionPgModel = new CollectionPgModel(); @@ -134,11 +109,36 @@ test.before(async (t) => { t.context.testPgExecution ); t.context.executionCumulusId = pgExecution.cumulus_id; + const timestamp = new Date(); + t.context.pdrs = range(2).map(() => fakePdrRecordFactory({ + collection_cumulus_id: t.context.collectionCumulusId, + provider_cumulus_id: t.context.providerCumulusId, + execution_cumulus_id: t.context.executionCumulusId, + progress: 0.5, + pan_sent: false, + pan_message: `pan${cryptoRandomString({ length: 10 })}`, + stats: { + processing: 0, + completed: 0, + failed: 0, + total: 0, + }, + address: `address${cryptoRandomString({ length: 10 })}`, + original_url: 'https://example.com', + duration: 6.8, + created_at: timestamp, + updated_at: timestamp, + })); + + t.context.pdrPgModel = new PdrPgModel(); + t.context.pgPdrs = await t.context.pdrPgModel.insert( + knex, + t.context.pdrs + ); }); test.after.always(async (t) => { await accessTokenModel.deleteTable(); - await cleanupTestIndex(t.context); await recursivelyDeleteS3Bucket(process.env.system_bucket); await destroyLocalTestDb({ knex: t.context.knex, @@ -208,7 +208,7 @@ test('CUMULUS-912 DELETE with pathParameters and with an invalid access token re test.todo('CUMULUS-912 DELETE with pathParameters and with an unauthorized user returns an unauthorized response'); -test('default returns list of pdrs', async (t) => { +test.serial('default returns list of pdrs', async (t) => { const response = await request(app) .get('/pdrs') .set('Accept', 'application/json') @@ -218,15 +218,15 @@ test('default returns list of pdrs', async (t) => { const { meta, results } = response.body; t.is(results.length, 2); t.is(meta.stack, process.env.stackName); - t.is(meta.table, 'pdr'); + t.is(meta.table, 'pdrs'); t.is(meta.count, 2); - const pdrNames = fakePdrs.map((i) => i.pdrName); + const pdrNames = t.context.pdrs.map((i) => i.name); results.forEach((r) => { t.true(pdrNames.includes(r.pdrName)); }); }); -test('GET returns an existing pdr', async (t) => { +test.serial('GET returns an existing pdr', async (t) => { const timestamp = new Date(); const newPGPdr = { @@ -281,7 +281,7 @@ test('GET fails if pdr is not found', async (t) => { t.true(message.includes('No record found for')); }); -test('DELETE returns a 404 if PostgreSQL and Elasticsearch PDR cannot be found', async (t) => { +test('DELETE returns a 404 if PostgreSQL PDR cannot be found', async (t) => { const nonExistentPdr = fakePdrFactory('completed'); const response = await request(app) .delete(`/pdrs/${nonExistentPdr.pdrName}`) @@ -291,9 +291,8 @@ test('DELETE returns a 404 if PostgreSQL and Elasticsearch PDR cannot be found', t.is(response.body.message, 'No record found'); }); -test('Deleting a PDR that exists in PostgreSQL and not Elasticsearch succeeds', async (t) => { +test.serial('Deleting a PDR that exists in PostgreSQL succeeds', async (t) => { const { - esPdrsClient, collectionCumulusId, providerCumulusId, knex, @@ -310,12 +309,6 @@ test('Deleting a PDR that exists in PostgreSQL and not Elasticsearch succeeds', knex, { cumulus_id: pgPdr.cumulus_id } ); - t.false( - await esPdrsClient.exists( - originalPgRecord.name - ) - ); - const response = await request(app) .delete(`/pdrs/${originalPgRecord.name}`) .set('Accept', 'application/json') @@ -327,43 +320,10 @@ test('Deleting a PDR that exists in PostgreSQL and not Elasticsearch succeeds', t.false(await pdrPgModel.exists(knex, { name: originalPgRecord.name })); }); -test.serial('Deleting a PDR that exists in Elastisearch and not PostgreSQL succeeds', async (t) => { - const { - esPdrsClient, - testPgCollection, - testPgProvider, - knex, - pdrPgModel, - } = t.context; - - const testPdr = fakePdrFactoryV2({ - collectionId: constructCollectionId(testPgCollection.name, testPgCollection.version), - provider: testPgProvider.name, - }); - await indexer.indexPdr(t.context.esClient, testPdr, t.context.esIndex); - - t.false(await pdrPgModel.exists(knex, { name: testPdr.pdrName })); - - const response = await request(app) - .delete(`/pdrs/${testPdr.pdrName}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - const { detail } = response.body; - - t.is(detail, 'Record deleted'); - t.false( - await esPdrsClient.exists( - testPdr.pdrName - ) - ); -}); - test.serial('DELETE handles the case where the PDR exists in PostgreSQL but not in S3', async (t) => { const { knex, pdrPgModel, - esClient, collectionCumulusId, providerCumulusId, } = t.context; @@ -377,9 +337,6 @@ test.serial('DELETE handles the case where the PDR exists in PostgreSQL but not const originalPgRecord = await pdrPgModel.get( knex, { cumulus_id: pdr.cumulus_id } ); - const originalPdr = await translatePostgresPdrToApiPdr(originalPgRecord, knex); - await indexer.indexPdr(esClient, originalPdr, process.env.ES_INDEX); - const response = await request(app) .delete(`/pdrs/${originalPgRecord.name}`) .set('Accept', 'application/json') @@ -387,14 +344,12 @@ test.serial('DELETE handles the case where the PDR exists in PostgreSQL but not .expect(200); t.is(response.status, 200); - const parsedBody = response.body; t.is(parsedBody.detail, 'Record deleted'); t.false(await pdrPgModel.exists(knex, { name: originalPgRecord.name })); - t.false(await t.context.esPdrsClient.exists(originalPgRecord.name)); }); -test.serial('DELETE removes a PDR from all data stores', async (t) => { +test.serial('DELETE removes a PDR from data store', async (t) => { const { originalPgRecord, } = await createPdrTestRecords(t.context); @@ -408,11 +363,6 @@ test.serial('DELETE removes a PDR from all data stores', async (t) => { t.is(detail, 'Record deleted'); t.false(await t.context.pdrPgModel.exists(t.context.knex, { name: originalPgRecord.name })); - t.false( - await t.context.esPdrsClient.exists( - originalPgRecord.name - ) - ); t.false( await s3ObjectExists({ Bucket: process.env.system_bucket, @@ -421,7 +371,7 @@ test.serial('DELETE removes a PDR from all data stores', async (t) => { ); }); -test.serial('del() does not remove from Elasticsearch/S3 if removing from PostgreSQL fails', async (t) => { +test.serial('del() does not remove from S3 if removing from PostgreSQL fails', async (t) => { const { originalPgRecord, } = await createPdrTestRecords( @@ -432,12 +382,6 @@ test.serial('del() does not remove from Elasticsearch/S3 if removing from Postgr await t.context.pdrPgModel.delete(t.context.knex, { name: originalPgRecord.name, }); - await indexer.deleteRecord({ - esClient: t.context.esClient, - id: originalPgRecord.name, - type: 'pdr', - index: t.context.esIndex, - }); await deleteS3Object(process.env.system_bucket, pdrS3Key(originalPgRecord.name)); }); @@ -470,11 +414,6 @@ test.serial('del() does not remove from Elasticsearch/S3 if removing from Postgr name: originalPgRecord.name, }) ); - t.true( - await t.context.esPdrsClient.exists( - originalPgRecord.name - ) - ); t.true( await s3ObjectExists({ Bucket: process.env.system_bucket, @@ -483,7 +422,7 @@ test.serial('del() does not remove from Elasticsearch/S3 if removing from Postgr ); }); -test.serial('del() does not remove from PostgreSQL/S3 if removing from Elasticsearch fails', async (t) => { +test.serial('del() does not remove from PostgreSQL if removing from S3 fails', async (t) => { const { originalPgRecord, } = await createPdrTestRecords( @@ -494,76 +433,6 @@ test.serial('del() does not remove from PostgreSQL/S3 if removing from Elasticse await t.context.pdrPgModel.delete(t.context.knex, { name: originalPgRecord.name, }); - await indexer.deleteRecord({ - esClient: t.context.esClient, - id: originalPgRecord.name, - type: 'pdr', - index: t.context.esIndex, - }); - await deleteS3Object(process.env.system_bucket, pdrS3Key(originalPgRecord.name)); - }); - - const fakeEsClient = { - initializeEsClient: () => Promise.resolve(), - client: { - delete: () => { - throw new Error('something bad'); - }, - }, - }; - - const expressRequest = { - params: { - pdrName: originalPgRecord.name, - }, - testContext: { - knex: t.context.knex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - del(expressRequest, response), - { message: 'something bad' } - ); - - t.true( - await t.context.pdrPgModel.exists(t.context.knex, { - name: originalPgRecord.name, - }) - ); - t.true( - await t.context.esPdrsClient.exists( - originalPgRecord.name - ) - ); - t.true( - await s3ObjectExists({ - Bucket: process.env.system_bucket, - Key: pdrS3Key(originalPgRecord.name), - }) - ); -}); - -test.serial('del() does not remove from PostgreSQL/Elasticsearch if removing from S3 fails', async (t) => { - const { - originalPgRecord, - } = await createPdrTestRecords( - t.context - ); - - t.teardown(async () => { - await t.context.pdrPgModel.delete(t.context.knex, { - name: originalPgRecord.name, - }); - await indexer.deleteRecord({ - esClient: t.context.esClient, - id: originalPgRecord.name, - type: 'pdr', - index: t.context.esIndex, - }); await deleteS3Object(process.env.system_bucket, pdrS3Key(originalPgRecord.name)); }); @@ -595,11 +464,6 @@ test.serial('del() does not remove from PostgreSQL/Elasticsearch if removing fro name: originalPgRecord.name, }) ); - t.true( - await t.context.esPdrsClient.exists( - originalPgRecord.name - ) - ); t.true( await s3ObjectExists({ Bucket: process.env.system_bucket, diff --git a/packages/api/tests/endpoints/test-reconciliation-reports.js b/packages/api/tests/endpoints/test-reconciliation-reports.js index fd9a4f39533..e22a9302121 100644 --- a/packages/api/tests/endpoints/test-reconciliation-reports.js +++ b/packages/api/tests/endpoints/test-reconciliation-reports.js @@ -7,8 +7,17 @@ const isEqual = require('lodash/isEqual'); const isMatch = require('lodash/isMatch'); const omit = require('lodash/omit'); const request = require('supertest'); +const cryptoRandomString = require('crypto-random-string'); -const { localStackConnectionEnv } = require('@cumulus/db'); +const { + ReconciliationReportPgModel, + generateLocalTestDb, + destroyLocalTestDb, + localStackConnectionEnv, + migrationDir, + fakeReconciliationReportRecordFactory, + translatePostgresReconReportToApiReconReport, +} = require('@cumulus/db'); const awsServices = require('@cumulus/aws-client/services'); const { buildS3Uri, @@ -17,14 +26,10 @@ const { recursivelyDeleteS3Bucket, } = require('@cumulus/aws-client/S3'); const { randomId } = require('@cumulus/common/test-utils'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const indexer = require('@cumulus/es-client/indexer'); -const { getEsClient } = require('@cumulus/es-client/search'); const startAsyncOperation = require('../../lib/startAsyncOperation'); const { createFakeJwtAuthToken, - fakeReconciliationReportFactory, setAuthorizedOAuthUsers, } = require('../../lib/testUtils'); const assertions = require('../../lib/assertions'); @@ -35,7 +40,6 @@ process.env.invoke = 'granule-reconciliation-reports'; process.env.stackName = 'test-stack'; process.env.system_bucket = 'testsystembucket'; process.env.AccessTokensTable = randomId('accessTokensTable'); -process.env.ReconciliationReportsTable = randomId('recReportsTable'); process.env.TOKEN_SECRET = randomId('tokenSecret'); process.env.stackName = randomId('stackname'); process.env.system_bucket = randomId('bucket'); @@ -44,43 +48,22 @@ process.env.AsyncOperationTaskDefinition = randomId('asyncOpTaskDefinition'); process.env.EcsCluster = randomId('ecsCluster'); // import the express app after setting the env variables -const { - app, -} = require('../../app'); -const { - createReport, -} = require('../../endpoints/reconciliation-reports'); +const { app } = require('../../app'); +const { createReport } = require('../../endpoints/reconciliation-reports'); const { normalizeEvent } = require('../../lib/reconciliationReport/normalizeEvent'); const { buildFakeExpressResponse } = require('./utils'); -let esClient; -const esIndex = randomId('esindex'); +const testDbName = `test_recon_reports_${cryptoRandomString({ length: 10 })}`; let jwtAuthToken; let accessTokenModel; -let reconciliationReportModel; let fakeReportRecords = []; -test.before(async () => { - // create esClient - esClient = await getEsClient('fakehost'); - - const esAlias = randomId('esalias'); - process.env.ES_INDEX = esAlias; - - // add fake elasticsearch index - await bootstrapElasticSearch({ - host: 'fakehost', - index: esIndex, - alias: esAlias, - }); +test.before(async (t) => { accessTokenModel = new models.AccessToken(); await accessTokenModel.createTable(); - reconciliationReportModel = new models.ReconciliationReport(); - await reconciliationReportModel.createTable(); - await awsServices.s3().createBucket({ Bucket: process.env.system_bucket, }); @@ -93,6 +76,17 @@ test.before(async () => { username, }); + const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); + t.context.knex = knex; + t.context.knexAdmin = knexAdmin; + process.env = { + ...process.env, + ...localStackConnectionEnv, + PG_DATABASE: testDbName, + }; + + t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); + const reportNameTypes = [ { name: randomId('report1'), type: 'Inventory' }, { name: randomId('report2'), type: 'Granule Inventory' }, @@ -102,7 +96,7 @@ test.before(async () => { const reportDirectory = `${process.env.stackName}/reconciliation-reports`; const typeToExtension = (type) => ((type === 'Granule Inventory') ? '.csv' : '.json'); - fakeReportRecords = reportNameTypes.map((nameType) => fakeReconciliationReportFactory({ + fakeReportRecords = reportNameTypes.map((nameType) => fakeReconciliationReportRecordFactory({ name: nameType.name, type: nameType.type, location: buildS3Uri(process.env.system_bucket, @@ -119,19 +113,17 @@ test.before(async () => { }), }))); - // add records to es - await Promise.all(fakeReportRecords.map((reportRecord) => - reconciliationReportModel.create(reportRecord) - .then((record) => indexer.indexReconciliationReport(esClient, record, esAlias)))); + await t.context.reconciliationReportPgModel.insert(t.context.knex, fakeReportRecords); }); -test.after.always(async () => { +test.after.always(async (t) => { await accessTokenModel.deleteTable(); - await reconciliationReportModel.deleteTable(); - await esClient.client.indices.delete({ - index: esIndex, - }); await recursivelyDeleteS3Bucket(process.env.system_bucket); + await destroyLocalTestDb({ + knex: t.context.knex, + knexAdmin: t.context.knexAdmin, + testDbName, + }); }); test.serial('CUMULUS-911 GET without pathParameters and without an Authorization header returns an Authorization Missing response', async (t) => { @@ -231,8 +223,13 @@ test.serial('default returns list of reports', async (t) => { const recordsAreEqual = (record1, record2) => isEqual(omit(record1, ['updatedAt', 'timestamp']), omit(record2, ['updatedAt', 'timestamp'])); + // fakeReportRecords were created with the factory that creates PG version recon reports, so + // should be translated as the list endpoint returns the API version of recon reports + const fakeReportApiRecords = fakeReportRecords.map((fakeRecord) => + translatePostgresReconReportToApiReconReport(fakeRecord)); + results.results.forEach((item) => { - const recordsFound = fakeReportRecords.filter((record) => recordsAreEqual(record, item)); + const recordsFound = fakeReportApiRecords.filter((record) => recordsAreEqual(record, item)); t.is(recordsFound.length, 1); }); }); diff --git a/packages/api/tests/endpoints/test-rules.js b/packages/api/tests/endpoints/test-rules.js index 20a0a222534..afdec634c3c 100644 --- a/packages/api/tests/endpoints/test-rules.js +++ b/packages/api/tests/endpoints/test-rules.js @@ -16,11 +16,9 @@ const { mockClient } = require('aws-sdk-client-mock'); const { createSnsTopic } = require('@cumulus/aws-client/SNS'); const { randomString, randomId } = require('@cumulus/common/test-utils'); +const { removeNilProperties } = require('@cumulus/common/util'); const workflows = require('@cumulus/common/workflows'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); + const { CollectionPgModel, destroyLocalTestDb, @@ -39,8 +37,6 @@ const { } = require('@cumulus/db'); const awsServices = require('@cumulus/aws-client/services'); const S3 = require('@cumulus/aws-client/S3'); -const { Search } = require('@cumulus/es-client/search'); -const indexer = require('@cumulus/es-client/indexer'); const { constructCollectionId } = require('@cumulus/message/Collections'); const { @@ -58,7 +54,7 @@ const { createRuleTestRecords, createSqsQueues, } = require('../../lib/testUtils'); -const { patch, post, put, del } = require('../../endpoints/rules'); +const { patch, post, put } = require('../../endpoints/rules'); const rulesHelpers = require('../../lib/rulesHelpers'); const AccessToken = require('../../models/access-tokens'); @@ -116,16 +112,6 @@ test.before(async (t) => { t.context.testKnex = knex; t.context.testKnexAdmin = knexAdmin; - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esRulesClient = new Search( - {}, - 'rule', - t.context.esIndex - ); - process.env.ES_INDEX = esIndex; - await S3.createBucket(process.env.system_bucket); buildPayloadStub = setBuildPayloadStub(); @@ -174,6 +160,20 @@ test.before(async (t) => { provider: t.context.pgProvider.name, }); + t.context.testRuleWithoutForeignKeys = fakeRuleFactoryV2({ + name: 'testRuleWithoutForeignKeys', + workflow: workflow, + rule: { + type: 'onetime', + arn: 'arn', + value: 'value', + }, + state: 'ENABLED', + queueUrl: 'https://sqs.us-west-2.amazonaws.com/123456789012/queue_url', + collection: undefined, + provider: undefined, + }); + const username = randomString(); await setAuthorizedOAuthUsers([username]); @@ -199,8 +199,13 @@ test.before(async (t) => { const ruleWithTrigger = await rulesHelpers.createRuleTrigger(t.context.testRule); t.context.collectionId = constructCollectionId(collectionName, collectionVersion); t.context.testPgRule = await translateApiRuleToPostgresRuleRaw(ruleWithTrigger, knex); - await indexer.indexRule(esClient, ruleWithTrigger, t.context.esIndex); t.context.rulePgModel.create(knex, t.context.testPgRule); + + const rule2WithTrigger = await rulesHelpers.createRuleTrigger( + t.context.testRuleWithoutForeignKeys + ); + t.context.testPgRule2 = await translateApiRuleToPostgresRuleRaw(rule2WithTrigger, knex); + t.context.rulePgModel.create(knex, t.context.testPgRule2); }); test.beforeEach((t) => { @@ -215,7 +220,6 @@ test.beforeEach((t) => { test.after.always(async (t) => { await accessTokenModel.deleteTable(); await S3.recursivelyDeleteS3Bucket(process.env.system_bucket); - await cleanupTestIndex(t.context); buildPayloadStub.restore(); await destroyLocalTestDb({ @@ -383,7 +387,7 @@ test.serial('default returns list of rules', async (t) => { .expect(200); const { results } = response.body; - t.is(results.length, 1); + t.is(results.length, 2); }); test.serial('search returns correct list of rules', async (t) => { @@ -394,7 +398,7 @@ test.serial('search returns correct list of rules', async (t) => { .expect(200); const { results } = response.body; - t.is(results.length, 1); + t.is(results.length, 2); const newResponse = await request(app) .get('/rules?page=1&rule.type=sqs&state=ENABLED') @@ -406,7 +410,46 @@ test.serial('search returns correct list of rules', async (t) => { t.is(newResults.length, 0); }); -test('GET gets a rule', async (t) => { +test.serial('Rules search returns the expected fields', async (t) => { + const response = await request(app) + .get(`/rules?page=1&rule.type=onetime&provider=${t.context.pgProvider.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .expect(200); + + const { results } = response.body; + + const expectedRule = { + ...t.context.testRule, + updatedAt: results[0].updatedAt, + createdAt: results[0].createdAt, + }; + + t.is(results.length, 1); + t.deepEqual(results[0], expectedRule); +}); + +test.serial('Rules search returns results without a provider or collection', async (t) => { + const response = await request(app) + .get(`/rules?page=1&name=${t.context.testPgRule2.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .expect(200); + + const { results } = response.body; + + t.is(results.length, 1); + + const expectedRule = { + ...t.context.testRuleWithoutForeignKeys, + updatedAt: results[0].updatedAt, + createdAt: results[0].createdAt, + }; + + t.deepEqual(results[0], removeNilProperties(expectedRule)); +}); + +test.serial('GET gets a rule', async (t) => { const response = await request(app) .get(`/rules/${t.context.testRule.name}`) .set('Accept', 'application/json') @@ -477,12 +520,11 @@ test('403 error when calling the API endpoint to delete an existing rule without t.deepEqual(response.body, record); }); -test('POST creates a rule in all data stores', async (t) => { +test('POST creates a rule', async (t) => { const { collectionPgModel, newRule, providerPgModel, - rulePgModel, testKnex, } = t.context; @@ -525,19 +567,11 @@ test('POST creates a rule in all data stores', async (t) => { .expect(200); const { message } = response.body; - const fetchedPostgresRecord = await rulePgModel - .get(testKnex, { name: newRule.name }); t.is(message, 'Record saved'); - const translatedPgRecord = await translatePostgresRuleToApiRule(fetchedPostgresRecord, testKnex); - - const esRecord = await t.context.esRulesClient.get( - newRule.name - ); - t.like(esRecord, translatedPgRecord); }); -test.serial('post() creates SNS rule with same trigger information in PostgreSQL/Elasticsearch', async (t) => { +test.serial('post() creates SNS rule with trigger information in PostgreSQL', async (t) => { const { pgProvider, pgCollection, @@ -569,33 +603,18 @@ test.serial('post() creates SNS rule with same trigger information in PostgreSQL const pgRule = await t.context.rulePgModel .get(t.context.testKnex, { name: rule.name }); - const esRule = await t.context.esRulesClient.get( - rule.name - ); t.truthy(pgRule.arn); - t.truthy(esRule.rule.arn); - t.like( - esRule, - { - rule: { - type: 'sns', - value: topic1.TopicArn, - arn: pgRule.arn, - }, - } - ); t.like(pgRule, { name: rule.name, enabled: true, type: 'sns', - arn: esRule.rule.arn, value: topic1.TopicArn, }); }); -test.serial('post() creates the same Kinesis rule with trigger information in PostgreSQL/Elasticsearch', async (t) => { +test.serial('post() creates Kinesis rule with trigger information in PostgreSQL', async (t) => { const { pgProvider, pgCollection, @@ -627,24 +646,9 @@ test.serial('post() creates the same Kinesis rule with trigger information in Po const pgRule = await t.context.rulePgModel .get(t.context.testKnex, { name: rule.name }); - const esRule = await t.context.esRulesClient.get( - rule.name - ); - t.truthy(esRule.rule.arn); - t.truthy(esRule.rule.logEventArn); t.truthy(pgRule.arn); t.truthy(pgRule.log_event_arn); - t.like( - esRule, - { - ...rule, - rule: { - type: 'kinesis', - value: kinesisArn1, - }, - } - ); t.like(pgRule, { name: rule.name, enabled: true, @@ -653,7 +657,7 @@ test.serial('post() creates the same Kinesis rule with trigger information in Po }); }); -test.serial('post() creates the SQS rule with trigger information in PostgreSQL/Elasticsearch', async (t) => { +test.serial('post() creates the SQS rule with trigger information in PostgreSQL', async (t) => { const { pgProvider, pgCollection, @@ -691,20 +695,7 @@ test.serial('post() creates the SQS rule with trigger information in PostgreSQL/ const pgRule = await t.context.rulePgModel .get(t.context.testKnex, { name: rule.name }); - const esRule = await t.context.esRulesClient.get( - rule.name - ); - t.like( - esRule, - { - rule: { - type: 'sqs', - value: queueUrl1, - }, - meta: expectedMeta, - } - ); t.like(pgRule, { name: rule.name, enabled: true, @@ -714,7 +705,7 @@ test.serial('post() creates the SQS rule with trigger information in PostgreSQL/ }); }); -test.serial('post() creates the SQS rule with the meta provided in PostgreSQL/Elasticsearch', async (t) => { +test.serial('post() creates the SQS rule with the meta provided in PostgreSQL', async (t) => { const { pgProvider, pgCollection, @@ -756,20 +747,7 @@ test.serial('post() creates the SQS rule with the meta provided in PostgreSQL/El const pgRule = await t.context.rulePgModel .get(t.context.testKnex, { name: rule.name }); - const esRule = await t.context.esRulesClient.get( - rule.name - ); - t.like( - esRule, - { - rule: { - type: 'sqs', - value: queueUrl1, - }, - meta: expectedMeta, - } - ); t.like(pgRule, { name: rule.name, enabled: true, @@ -965,67 +943,8 @@ test.serial('POST returns a 500 response if record creation throws unexpected er } }); -test.serial('post() does not write to Elasticsearch if writing to PostgreSQL fails', async (t) => { - const { newRule, testKnex } = t.context; - - const fakeRulePgModel = { - create: () => { - throw new Error('something bad'); - }, - }; - - const expressRequest = { - body: newRule, - testContext: { - knex: testKnex, - rulePgModel: fakeRulePgModel, - }, - }; - const response = buildFakeExpressResponse(); - await post(expressRequest, response); - - const dbRecords = await t.context.rulePgModel - .search(t.context.testKnex, { name: newRule.name }); - - t.is(dbRecords.length, 0); - t.false(await t.context.esRulesClient.exists( - newRule.name - )); -}); - -test.serial('post() does not write to PostgreSQL if writing to Elasticsearch fails', async (t) => { - const { newRule, testKnex } = t.context; - - const fakeEsClient = { - client: { - index: () => Promise.reject(new Error('something bad')), - }, - }; - - const expressRequest = { - body: newRule, - testContext: { - knex: testKnex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await post(expressRequest, response); - - const dbRecords = await t.context.rulePgModel - .search(t.context.testKnex, { name: newRule.name }); - - t.is(dbRecords.length, 0); - t.false(await t.context.esRulesClient.exists( - newRule.name - )); -}); - -test.serial('PATCH updates an existing rule in all data stores', async (t) => { +test.serial('PATCH updates an existing rule', async (t) => { const { - esRulesClient, rulePgModel, testKnex, } = t.context; @@ -1039,7 +958,6 @@ test.serial('PATCH updates an existing rule in all data stores', async (t) => { const { originalApiRule, originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( t.context, { @@ -1052,8 +970,6 @@ test.serial('PATCH updates an existing rule in all data stores', async (t) => { t.deepEqual(originalPgRecord.meta, oldMetaFields); t.is(originalPgRecord.payload, null); - t.deepEqual(originalEsRecord.meta, oldMetaFields); - t.is(originalEsRecord.payload, undefined); const updateMetaFields = { nestedFieldOne: { @@ -1085,25 +1001,10 @@ test.serial('PATCH updates an existing rule in all data stores', async (t) => { .expect(200); const actualPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); - const updatedEsRecord = await esRulesClient.get(originalApiRule.name); const expectedMeta = merge(cloneDeep(oldMetaFields), updateMetaFields); - // PG and ES records have the same timestamps + // PG record has the original timestamp t.true(actualPostgresRule.updated_at > originalPgRecord.updated_at); - t.is(actualPostgresRule.created_at.getTime(), updatedEsRecord.createdAt); - t.is(actualPostgresRule.updated_at.getTime(), updatedEsRecord.updatedAt); - t.deepEqual( - updatedEsRecord, - { - ...originalEsRecord, - state: 'ENABLED', - meta: expectedMeta, - payload: updatePayload, - createdAt: originalPgRecord.created_at.getTime(), - updatedAt: actualPostgresRule.updated_at.getTime(), - timestamp: updatedEsRecord.timestamp, - } - ); t.deepEqual( actualPostgresRule, { @@ -1117,9 +1018,8 @@ test.serial('PATCH updates an existing rule in all data stores', async (t) => { ); }); -test.serial('PATCH nullifies expected fields for existing rule in all datastores', async (t) => { +test.serial('PATCH nullifies expected fields for existing rule', async (t) => { const { - esRulesClient, rulePgModel, testKnex, } = t.context; @@ -1168,7 +1068,6 @@ test.serial('PATCH nullifies expected fields for existing rule in all datastores .expect(200); const actualPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); - const updatedEsRecord = await esRulesClient.get(originalApiRule.name); const apiRule = await translatePostgresRuleToApiRule(actualPostgresRule, testKnex); const expectedApiRule = { @@ -1180,13 +1079,6 @@ test.serial('PATCH nullifies expected fields for existing rule in all datastores }; t.deepEqual(apiRule, expectedApiRule); - const expectedEsRecord = { - ...expectedApiRule, - _id: updatedEsRecord._id, - timestamp: updatedEsRecord.timestamp, - }; - t.deepEqual(updatedEsRecord, expectedEsRecord); - t.deepEqual( actualPostgresRule, { @@ -1231,13 +1123,11 @@ test.serial('PATCH sets SNS rule to "disabled" and removes source mapping ARN', }); const { - esRulesClient, rulePgModel, testKnex, } = t.context; const { originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( t.context, { @@ -1249,7 +1139,6 @@ test.serial('PATCH sets SNS rule to "disabled" and removes source mapping ARN', ); t.truthy(originalPgRecord.arn); - t.is(originalEsRecord.rule.arn, originalPgRecord.arn); const updateRule = { name: originalPgRecord.name, @@ -1264,10 +1153,8 @@ test.serial('PATCH sets SNS rule to "disabled" and removes source mapping ARN', .expect(200); const updatedPostgresRule = await rulePgModel.get(testKnex, { name: originalPgRecord.name }); - const updatedEsRecord = await esRulesClient.get(originalPgRecord.name); t.is(updatedPostgresRule.arn, null); - t.is(updatedEsRecord.rule.arn, undefined); }); test('PATCH returns 404 for non-existent rule', async (t) => { @@ -1441,120 +1328,7 @@ test('PATCH returns a 400 response if rule value is not specified for non-onetim t.truthy(message.match(regexp)); }); -test('PATCH does not write to Elasticsearch if writing to PostgreSQL fails', async (t) => { - const { testKnex } = t.context; - const { - originalApiRule, - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'queue-1', - workflow, - } - ); - - const fakerulePgModel = { - get: () => Promise.resolve(originalPgRecord), - upsert: () => Promise.reject(new Error('something bad')), - }; - - const updatedRule = { - name: originalApiRule.name, - queueUrl: 'queue-2', - }; - - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - body: updatedRule, - testContext: { - knex: testKnex, - rulePgModel: fakerulePgModel, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - patch(expressRequest, response), - { message: 'something bad' } - ); - - t.deepEqual( - await t.context.rulePgModel.get(t.context.testKnex, { - name: originalPgRecord.name, - }), - originalPgRecord - ); - t.deepEqual( - await t.context.esRulesClient.get( - originalPgRecord.name - ), - originalEsRecord - ); -}); - -test('PATCH does not write to PostgreSQL if writing to Elasticsearch fails', async (t) => { - const { testKnex } = t.context; - const { - originalApiRule, - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'queue-1', - workflow, - } - ); - - const fakeEsClient = { - client: { - index: () => Promise.reject(new Error('something bad')), - }, - }; - - const updatedRule = { - name: originalApiRule.name, - queueUrl: 'queue-2', - }; - - const expressRequest = { - params: { - name: originalApiRule.name, - }, - body: updatedRule, - testContext: { - knex: testKnex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - patch(expressRequest, response), - { message: 'something bad' } - ); - - t.deepEqual( - await t.context.rulePgModel.get(t.context.testKnex, { - name: originalApiRule.name, - }), - originalPgRecord - ); - t.deepEqual( - await t.context.esRulesClient.get( - originalApiRule.name - ), - originalEsRecord - ); -}); - -test.serial('PATCH creates the same updated SNS rule in PostgreSQL/Elasticsearch', async (t) => { +test.serial('PATCH keeps initial trigger information if writing to PostgreSQL fails', async (t) => { const { pgProvider, pgCollection, @@ -1565,12 +1339,12 @@ test.serial('PATCH creates the same updated SNS rule in PostgreSQL/Elasticsearch const { originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( - t.context, + { + ...t.context, + }, { workflow, - queueUrl: 'fake-queue-url', state: 'ENABLED', type: 'sns', value: topic1.TopicArn, @@ -1582,8 +1356,8 @@ test.serial('PATCH creates the same updated SNS rule in PostgreSQL/Elasticsearch } ); - t.truthy(originalEsRecord.rule.value); t.truthy(originalPgRecord.value); + const updateRule = { name: originalPgRecord.name, rule: { @@ -1597,643 +1371,86 @@ test.serial('PATCH creates the same updated SNS rule in PostgreSQL/Elasticsearch name: originalPgRecord.name, }, body: updateRule, + testContext: { + rulePgModel: { + get: () => Promise.resolve(originalPgRecord), + upsert: () => { + throw new Error('PG fail'); + }, + }, + }, }; + const response = buildFakeExpressResponse(); - await patch(expressRequest, response); - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name + + await t.throwsAsync( + patch(expressRequest, response), + { message: 'PG fail' } ); - t.truthy(updatedEsRule.rule.value); - t.truthy(updatedPgRule.value); + const updatedPgRule = await t.context.rulePgModel + .get(t.context.testKnex, { name: updateRule.name }); - t.not(updatedEsRule.rule.value, originalEsRecord.rule.value); - t.not(updatedPgRule.value, originalPgRecord.value); + t.is(updatedPgRule.arn, originalPgRecord.arn); - t.deepEqual( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sns', - value: topic2.TopicArn, - }, - } - ); - t.deepEqual(updatedPgRule, { + t.like(updatedPgRule, { ...originalPgRecord, updated_at: updatedPgRule.updated_at, type: 'sns', - arn: updatedPgRule.arn, - value: topic2.TopicArn, + value: topic1.TopicArn, }); }); -test.serial('PATCH creates the same updated Kinesis rule in PostgreSQL/Elasticsearch', async (t) => { +test.serial('PUT replaces an existing rule', async (t) => { const { - pgProvider, - pgCollection, + rulePgModel, + testKnex, } = t.context; - - const kinesisArn1 = `arn:aws:kinesis:us-east-1:000000000000:${randomId('kinesis1_')}`; - const kinesisArn2 = `arn:aws:kinesis:us-east-1:000000000000:${randomId('kinesis2_')}`; + const oldMetaFields = { + nestedFieldOne: { + fieldOne: 'fieldone-data', + }, + }; const { + originalApiRule, originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( t.context, { + queue_url: 'fake-queue-url', workflow, - state: 'ENABLED', - type: 'kinesis', - value: kinesisArn1, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, + meta: oldMetaFields, + tags: ['tag1', 'tag2'], } ); - t.truthy(originalEsRecord.rule.arn); - t.truthy(originalEsRecord.rule.logEventArn); - t.truthy(originalPgRecord.arn); - t.truthy(originalPgRecord.log_event_arn); + t.deepEqual(originalPgRecord.meta, oldMetaFields); + t.is(originalPgRecord.payload, null); - const updateRule = { - name: originalPgRecord.name, - rule: { - type: 'kinesis', - value: kinesisArn2, + const updateMetaFields = { + nestedFieldOne: { + nestedFieldOneKey2: randomId('nestedFieldOneKey2'), + 'key.with.period': randomId('key.with.period'), }, - }; - - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - body: updateRule, - }; - - const response = buildFakeExpressResponse(); - - await patch(expressRequest, response); - - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name - ); - - t.truthy(updatedEsRule.rule.arn); - t.truthy(updatedEsRule.rule.logEventArn); - t.truthy(updatedPgRule.arn); - t.truthy(updatedPgRule.log_event_arn); - - t.not(originalEsRecord.rule.arn, updatedEsRule.rule.arn); - t.not(originalEsRecord.rule.logEventArn, updatedEsRule.rule.logEventArn); - t.not(originalPgRecord.arn, updatedPgRule.arn); - t.not(originalPgRecord.log_event_arn, updatedPgRule.log_event_arn); - - t.deepEqual( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - arn: updatedEsRule.rule.arn, - logEventArn: updatedEsRule.rule.logEventArn, - type: 'kinesis', - value: kinesisArn2, - }, - } - ); - t.deepEqual(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'kinesis', - value: kinesisArn2, - arn: updatedPgRule.arn, - log_event_arn: updatedPgRule.log_event_arn, - }); -}); - -test.serial('PATCH creates the same SQS rule in PostgreSQL/Elasticsearch', async (t) => { - const { - pgProvider, - pgCollection, - } = t.context; - - const queue1 = randomId('queue'); - const queue2 = randomId('queue'); - - const { queueUrl: queueUrl1 } = await createSqsQueues(queue1); - const { queueUrl: queueUrl2 } = await createSqsQueues(queue2, 4, '100'); - - const { - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - { - ...t.context, - }, - { - workflow, - name: randomId('rule'), - state: 'ENABLED', - type: 'sqs', - value: queueUrl1, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, - } - ); - - const expectedMeta = { - visibilityTimeout: 300, - retries: 3, - }; - - t.deepEqual(originalPgRecord.meta, expectedMeta); - t.deepEqual(originalEsRecord.meta, expectedMeta); - - const updateRule = { - name: originalPgRecord.name, - rule: { - type: 'sqs', - value: queueUrl2, - }, - meta: { - retries: 2, - visibilityTimeout: null, - }, - }; - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - body: updateRule, - }; - const response = buildFakeExpressResponse(); - await patch(expressRequest, response); - - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - updateRule.name - ); - - const expectedMetaUpdate = { - visibilityTimeout: 100, - retries: 2, - }; - - t.deepEqual( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sqs', - value: queueUrl2, - }, - meta: expectedMetaUpdate, - } - ); - t.deepEqual(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'sqs', - value: queueUrl2, - meta: expectedMetaUpdate, - }); -}); - -test.serial('PATCH keeps initial trigger information if writing to PostgreSQL fails', async (t) => { - const { - pgProvider, - pgCollection, - } = t.context; - - const topic1 = await createSnsTopic(randomId('topic1_')); - const topic2 = await createSnsTopic(randomId('topic2_')); - - const { - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - { - ...t.context, - }, - { - workflow, - state: 'ENABLED', - type: 'sns', - value: topic1.TopicArn, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, - } - ); - - t.truthy(originalEsRecord.rule.value); - t.truthy(originalPgRecord.value); - - const updateRule = { - name: originalPgRecord.name, - rule: { - type: 'sns', - value: topic2.TopicArn, - }, - }; - - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - body: updateRule, - testContext: { - rulePgModel: { - get: () => Promise.resolve(originalPgRecord), - upsert: () => { - throw new Error('PG fail'); - }, - }, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - patch(expressRequest, response), - { message: 'PG fail' } - ); - - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name - ); - - t.is(updatedEsRule.rule.arn, originalEsRecord.rule.arn); - t.is(updatedPgRule.arn, originalPgRecord.arn); - - t.like( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sns', - value: topic1.TopicArn, - }, - } - ); - t.like(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'sns', - value: topic1.TopicArn, - }); -}); - -test.serial('PATCH keeps initial trigger information if writing to Elasticsearch fails', async (t) => { - const { - pgProvider, - pgCollection, - } = t.context; - - const topic1 = await createSnsTopic(randomId('topic1_')); - const topic2 = await createSnsTopic(randomId('topic2_')); - - const { - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - { - ...t.context, - }, - { - workflow, - state: 'ENABLED', - type: 'sns', - value: topic1.TopicArn, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, - } - ); - - t.truthy(originalEsRecord.rule.value); - t.truthy(originalPgRecord.value); - - const updateRule = { - name: originalPgRecord.name, - rule: { - type: 'sns', - value: topic2.TopicArn, - }, - }; - - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - body: updateRule, - testContext: { - esClient: { - client: { - index: () => { - throw new Error('ES fail'); - }, - }, - }, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - patch(expressRequest, response), - { message: 'ES fail' } - ); - - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name - ); - - t.is(updatedEsRule.rule.arn, originalEsRecord.rule.arn); - t.is(updatedPgRule.arn, originalPgRecord.arn); - - t.like( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sns', - value: topic1.TopicArn, - }, - } - ); - t.like(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'sns', - value: topic1.TopicArn, - }); -}); - -test.serial('PUT replaces an existing rule in all data stores', async (t) => { - const { - esRulesClient, - rulePgModel, - testKnex, - } = t.context; - const oldMetaFields = { - nestedFieldOne: { - fieldOne: 'fieldone-data', - }, - }; - - const { - originalApiRule, - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'fake-queue-url', - workflow, - meta: oldMetaFields, - tags: ['tag1', 'tag2'], - } - ); - - t.deepEqual(originalPgRecord.meta, oldMetaFields); - t.is(originalPgRecord.payload, null); - t.deepEqual(originalEsRecord.meta, oldMetaFields); - t.is(originalEsRecord.payload, undefined); - - const updateMetaFields = { - nestedFieldOne: { - nestedFieldOneKey2: randomId('nestedFieldOneKey2'), - 'key.with.period': randomId('key.with.period'), - }, - nestedFieldTwo: { - nestedFieldTwoKey1: randomId('nestedFieldTwoKey1'), - }, - }; - const updatePayload = { - foo: 'bar', - }; - const updateTags = ['tag2', 'tag3']; - const removedFields = ['queueUrl', 'queue_url', 'provider', 'collection']; - const updateRule = { - ...omit(originalApiRule, removedFields), - state: 'ENABLED', - meta: updateMetaFields, - payload: updatePayload, - tags: updateTags, - // these timestamps should not get used - createdAt: Date.now(), - updatedAt: Date.now(), - }; - - await request(app) - .put(`/rules/${updateRule.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send(updateRule) - .expect(200); - - const actualPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); - const updatedEsRecord = await esRulesClient.get(originalApiRule.name); - - // PG and ES records have the same timestamps - t.true(actualPostgresRule.updated_at > originalPgRecord.updated_at); - t.is(actualPostgresRule.created_at.getTime(), updatedEsRecord.createdAt); - t.is(actualPostgresRule.updated_at.getTime(), updatedEsRecord.updatedAt); - t.deepEqual( - updatedEsRecord, - { - ...omit(originalEsRecord, removedFields), - state: 'ENABLED', - meta: updateMetaFields, - payload: updatePayload, - tags: updateTags, - createdAt: originalApiRule.createdAt, - updatedAt: updatedEsRecord.updatedAt, - timestamp: updatedEsRecord.timestamp, - } - ); - t.deepEqual( - actualPostgresRule, - { - ...omit(originalPgRecord, removedFields), - enabled: true, - meta: updateMetaFields, - payload: updatePayload, - tags: updateTags, - queue_url: null, - created_at: originalPgRecord.created_at, - updated_at: actualPostgresRule.updated_at, - } - ); -}); - -test.serial('PUT removes existing fields if not specified or set to null', async (t) => { - const { - esRulesClient, - rulePgModel, - testKnex, - } = t.context; - const oldMetaFields = { - nestedFieldOne: { - fieldOne: 'fieldone-data', - }, - }; - - const { - originalApiRule, - originalPgRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'fake-queue-url', - workflow, - meta: oldMetaFields, - execution_name_prefix: 'testRule', - payload: { foo: 'bar' }, - value: randomId('value'), - tags: ['tag1', 'tag2'], - } - ); - - const removedFields = ['provider', 'collection', 'payload', 'tags']; - const updateRule = { - ...omit(originalApiRule, removedFields), - executionNamePrefix: null, - meta: null, - queueUrl: null, - rule: { - type: originalApiRule.rule.type, - }, - createdAt: null, - updatedAt: null, - }; - - await request(app) - .put(`/rules/${updateRule.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send(updateRule) - .expect(200); - - const actualPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); - const updatedEsRecord = await esRulesClient.get(originalApiRule.name); - const apiRule = await translatePostgresRuleToApiRule(actualPostgresRule, testKnex); - - const expectedApiRule = { - ...pick(originalApiRule, ['name', 'workflow', 'createdAt', 'state']), - rule: { - type: originalApiRule.rule.type, - }, - updatedAt: apiRule.updatedAt, - }; - t.deepEqual(apiRule, expectedApiRule); - - const expectedEsRecord = { - ...expectedApiRule, - _id: updatedEsRecord._id, - timestamp: updatedEsRecord.timestamp, - }; - t.deepEqual(updatedEsRecord, expectedEsRecord); - - t.deepEqual( - actualPostgresRule, - { - ...originalPgRecord, - enabled: false, - execution_name_prefix: null, - meta: null, - payload: null, - queue_url: null, - type: originalApiRule.rule.type, - value: null, - tags: null, - created_at: originalPgRecord.created_at, - updated_at: actualPostgresRule.updated_at, - } - ); -}); - -test.serial('PUT sets SNS rule to "disabled" and removes source mapping ARN', async (t) => { - const snsMock = mockClient(awsServices.sns()); - - snsMock - .onAnyCommand() - .rejects() - .on(ListSubscriptionsByTopicCommand) - .resolves({ - Subscriptions: [{ - Endpoint: process.env.messageConsumer, - SubscriptionArn: randomString(), - }], - }) - .on(UnsubscribeCommand) - .resolves({}); - const mockLambdaClient = mockClient(awsServices.lambda()).onAnyCommand().rejects(); - mockLambdaClient.on(AddPermissionCommand).resolves(); - mockLambdaClient.on(RemovePermissionCommand).resolves(); - - t.teardown(() => { - snsMock.restore(); - mockLambdaClient.restore(); - }); - const { - esRulesClient, - rulePgModel, - testKnex, - } = t.context; - const { - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - t.context, - { - value: 'sns-arn', - type: 'sns', - enabled: true, - workflow, - } - ); - - t.truthy(originalPgRecord.arn); - t.is(originalEsRecord.rule.arn, originalPgRecord.arn); - - const translatedPgRecord = await translatePostgresRuleToApiRule(originalPgRecord, testKnex); - + nestedFieldTwo: { + nestedFieldTwoKey1: randomId('nestedFieldTwoKey1'), + }, + }; + const updatePayload = { + foo: 'bar', + }; + const updateTags = ['tag2', 'tag3']; + const removedFields = ['queueUrl', 'queue_url', 'provider', 'collection']; const updateRule = { - ...translatedPgRecord, - state: 'DISABLED', + ...omit(originalApiRule, removedFields), + state: 'ENABLED', + meta: updateMetaFields, + payload: updatePayload, + tags: updateTags, + // these timestamps should not get used + createdAt: Date.now(), + updatedAt: Date.now(), }; await request(app) @@ -2243,629 +1460,308 @@ test.serial('PUT sets SNS rule to "disabled" and removes source mapping ARN', as .send(updateRule) .expect(200); - const updatedPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); - const updatedEsRecord = await esRulesClient.get(translatedPgRecord.name); - - t.is(updatedPostgresRule.arn, null); - t.is(updatedEsRecord.rule.arn, undefined); -}); - -test('PUT returns 404 for non-existent rule', async (t) => { - const name = 'new_make_coffee'; - const response = await request(app) - .put(`/rules/${name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send({ name }) - .expect(404); - - const { message, record } = response.body; - t.true(message.includes(name)); - t.falsy(record); -}); - -test('PUT returns 400 for name mismatch between params and payload', - async (t) => { - const response = await request(app) - .put(`/rules/${randomString()}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send({ name: randomString() }) - .expect(400); - const { message, record } = response.body; - - t.true(message.includes('Expected rule name to be')); - t.falsy(record); - }); - -test('PUT returns a 400 response if record is missing workflow property', async (t) => { - const { - originalApiRule, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'fake-queue-url', - workflow, - } - ); - - // Set required property to null to trigger create error - originalApiRule.workflow = null; - - const response = await request(app) - .put(`/rules/${originalApiRule.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send(originalApiRule) - .expect(400); - const { message } = response.body; - t.true(message.includes('The record has validation errors. Rule workflow is undefined')); -}); - -test('PUT returns a 400 response if record is missing type property', async (t) => { - const { - originalApiRule, - originalPgRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'fake-queue-url', - workflow, - } - ); - originalApiRule.rule.type = null; - const response = await request(app) - .put(`/rules/${originalPgRecord.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send(originalApiRule) - .expect(400); - const { message } = response.body; - t.true(message.includes('The record has validation errors. Rule type is undefined.')); -}); - -test('PUT returns a 400 response if rule name is invalid', async (t) => { - const { - originalApiRule, - originalPgRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'fake-queue-url', - workflow, - } - ); - originalApiRule.name = 'bad rule name'; - const response = await request(app) - .put(`/rules/${originalPgRecord.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send(originalApiRule) - .expect(400); - const { message } = response.body; - t.true(message.includes(originalApiRule.name)); -}); - -test('PUT returns a 400 response if rule type is invalid', async (t) => { - const { - originalApiRule, - originalPgRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'fake-queue-url', - workflow, - } - ); - originalApiRule.rule.type = 'invalid'; - - const response = await request(app) - .put(`/rules/${originalPgRecord.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send(originalApiRule) - .expect(400); - - const { message } = response.body; - const regexp = new RegExp('The record has validation errors:.*rule.type.*should be equal to one of the allowed values'); - t.truthy(message.match(regexp)); -}); - -test('PUT returns a 400 response if rule value is not specified for non-onetime rule', async (t) => { - const { - originalApiRule, - originalPgRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'fake-queue-url', - workflow, - } - ); - originalApiRule.rule.type = 'kinesis'; - - const response = await request(app) - .put(`/rules/${originalPgRecord.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .send(originalApiRule) - .expect(400); - - const { message } = response.body; - const regexp = new RegExp('Rule value is undefined for kinesis rule'); - t.truthy(message.match(regexp)); -}); - -test('PUT does not write to Elasticsearch if writing to PostgreSQL fails', async (t) => { - const { testKnex } = t.context; - const { - originalApiRule, - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'queue-1', - workflow, - } - ); - - const fakerulePgModel = { - get: () => Promise.resolve(originalPgRecord), - upsert: () => Promise.reject(new Error('something bad')), - }; - - const updatedRule = { - ...originalApiRule, - queueUrl: 'queue-2', - }; - - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - body: updatedRule, - testContext: { - knex: testKnex, - rulePgModel: fakerulePgModel, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - put(expressRequest, response), - { message: 'something bad' } - ); - - t.deepEqual( - await t.context.rulePgModel.get(t.context.testKnex, { - name: originalPgRecord.name, - }), - originalPgRecord - ); - t.deepEqual( - await t.context.esRulesClient.get( - originalPgRecord.name - ), - originalEsRecord - ); -}); - -test('PUT does not write to PostgreSQL if writing to Elasticsearch fails', async (t) => { - const { testKnex } = t.context; - const { - originalApiRule, - originalPgRecord, - originalEsRecord, - } = await createRuleTestRecords( - t.context, - { - queue_url: 'queue-1', - workflow, - } - ); - - const fakeEsClient = { - client: { - index: () => Promise.reject(new Error('something bad')), - }, - }; - - const updatedRule = { - ...originalApiRule, - queueUrl: 'queue-2', - }; - - const expressRequest = { - params: { - name: originalApiRule.name, - }, - body: updatedRule, - testContext: { - knex: testKnex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - put(expressRequest, response), - { message: 'something bad' } - ); + const actualPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); + t.true(actualPostgresRule.updated_at > originalPgRecord.updated_at); t.deepEqual( - await t.context.rulePgModel.get(t.context.testKnex, { - name: originalApiRule.name, - }), - originalPgRecord - ); - t.deepEqual( - await t.context.esRulesClient.get( - originalApiRule.name - ), - originalEsRecord + actualPostgresRule, + { + ...omit(originalPgRecord, removedFields), + enabled: true, + meta: updateMetaFields, + payload: updatePayload, + tags: updateTags, + queue_url: null, + created_at: originalPgRecord.created_at, + updated_at: actualPostgresRule.updated_at, + } ); }); -test.serial('PUT creates the same updated SNS rule in PostgreSQL/Elasticsearch', async (t) => { +test.serial('PUT removes existing fields if not specified or set to null', async (t) => { const { - pgProvider, - pgCollection, + rulePgModel, + testKnex, } = t.context; - - const topic1 = await createSnsTopic(randomId('topic1_')); - const topic2 = await createSnsTopic(randomId('topic2_')); + const oldMetaFields = { + nestedFieldOne: { + fieldOne: 'fieldone-data', + }, + }; const { originalApiRule, originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( t.context, { + queue_url: 'fake-queue-url', workflow, - queueUrl: 'fake-queue-url', - state: 'ENABLED', - type: 'sns', - value: topic1.TopicArn, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, + meta: oldMetaFields, + execution_name_prefix: 'testRule', + payload: { foo: 'bar' }, + value: randomId('value'), + tags: ['tag1', 'tag2'], } ); - t.truthy(originalEsRecord.rule.value); - t.truthy(originalPgRecord.value); + const removedFields = ['provider', 'collection', 'payload', 'tags']; const updateRule = { - ...originalApiRule, + ...omit(originalApiRule, removedFields), + executionNamePrefix: null, + meta: null, + queueUrl: null, rule: { - type: 'sns', - value: topic2.TopicArn, + type: originalApiRule.rule.type, }, + createdAt: null, + updatedAt: null, }; - const expressRequest = { - params: { - name: originalApiRule.name, - }, - body: updateRule, - }; - const response = buildFakeExpressResponse(); - await put(expressRequest, response); - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name - ); + await request(app) + .put(`/rules/${updateRule.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(updateRule) + .expect(200); - t.truthy(updatedEsRule.rule.value); - t.truthy(updatedPgRule.value); + const actualPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); + const apiRule = await translatePostgresRuleToApiRule(actualPostgresRule, testKnex); - t.not(updatedEsRule.rule.value, originalEsRecord.rule.value); - t.not(updatedPgRule.value, originalPgRecord.value); + const expectedApiRule = { + ...pick(originalApiRule, ['name', 'workflow', 'createdAt', 'state']), + rule: { + type: originalApiRule.rule.type, + }, + updatedAt: apiRule.updatedAt, + }; + t.deepEqual(apiRule, expectedApiRule); t.deepEqual( - updatedEsRule, + actualPostgresRule, { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sns', - value: topic2.TopicArn, - }, + ...originalPgRecord, + enabled: false, + execution_name_prefix: null, + meta: null, + payload: null, + queue_url: null, + type: originalApiRule.rule.type, + value: null, + tags: null, + created_at: originalPgRecord.created_at, + updated_at: actualPostgresRule.updated_at, } ); - t.deepEqual(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'sns', - arn: updatedPgRule.arn, - value: topic2.TopicArn, - }); }); -test.serial('PUT creates the same updated Kinesis rule in PostgreSQL/Elasticsearch', async (t) => { - const { - pgProvider, - pgCollection, - } = t.context; +test.serial('PUT sets SNS rule to "disabled" and removes source mapping ARN', async (t) => { + const snsMock = mockClient(awsServices.sns()); - const kinesisArn1 = `arn:aws:kinesis:us-east-1:000000000000:${randomId('kinesis1_')}`; - const kinesisArn2 = `arn:aws:kinesis:us-east-1:000000000000:${randomId('kinesis2_')}`; + snsMock + .onAnyCommand() + .rejects() + .on(ListSubscriptionsByTopicCommand) + .resolves({ + Subscriptions: [{ + Endpoint: process.env.messageConsumer, + SubscriptionArn: randomString(), + }], + }) + .on(UnsubscribeCommand) + .resolves({}); + const mockLambdaClient = mockClient(awsServices.lambda()).onAnyCommand().rejects(); + mockLambdaClient.on(AddPermissionCommand).resolves(); + mockLambdaClient.on(RemovePermissionCommand).resolves(); + t.teardown(() => { + snsMock.restore(); + mockLambdaClient.restore(); + }); + const { + rulePgModel, + testKnex, + } = t.context; const { - originalApiRule, originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( t.context, { + value: 'sns-arn', + type: 'sns', + enabled: true, workflow, - state: 'ENABLED', - type: 'kinesis', - value: kinesisArn1, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, } ); - t.truthy(originalEsRecord.rule.arn); - t.truthy(originalEsRecord.rule.logEventArn); t.truthy(originalPgRecord.arn); - t.truthy(originalPgRecord.log_event_arn); + + const translatedPgRecord = await translatePostgresRuleToApiRule(originalPgRecord, testKnex); const updateRule = { - ...originalApiRule, - rule: { - type: 'kinesis', - value: kinesisArn2, - }, + ...translatedPgRecord, + state: 'DISABLED', }; - const expressRequest = { - params: { - name: originalApiRule.name, - }, - body: updateRule, - }; + await request(app) + .put(`/rules/${updateRule.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(updateRule) + .expect(200); - const response = buildFakeExpressResponse(); + const updatedPostgresRule = await rulePgModel.get(testKnex, { name: updateRule.name }); - await put(expressRequest, response); + t.is(updatedPostgresRule.arn, null); +}); - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name - ); +test('PUT returns 404 for non-existent rule', async (t) => { + const name = 'new_make_coffee'; + const response = await request(app) + .put(`/rules/${name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send({ name }) + .expect(404); + + const { message, record } = response.body; + t.true(message.includes(name)); + t.falsy(record); +}); - t.truthy(updatedEsRule.rule.arn); - t.truthy(updatedEsRule.rule.logEventArn); - t.truthy(updatedPgRule.arn); - t.truthy(updatedPgRule.log_event_arn); +test('PUT returns 400 for name mismatch between params and payload', + async (t) => { + const response = await request(app) + .put(`/rules/${randomString()}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send({ name: randomString() }) + .expect(400); + const { message, record } = response.body; - t.not(originalEsRecord.rule.arn, updatedEsRule.rule.arn); - t.not(originalEsRecord.rule.logEventArn, updatedEsRule.rule.logEventArn); - t.not(originalPgRecord.arn, updatedPgRule.arn); - t.not(originalPgRecord.log_event_arn, updatedPgRule.log_event_arn); + t.true(message.includes('Expected rule name to be')); + t.falsy(record); + }); - t.deepEqual( - updatedEsRule, +test('PUT returns a 400 response if record is missing workflow property', async (t) => { + const { + originalApiRule, + } = await createRuleTestRecords( + t.context, { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - arn: updatedEsRule.rule.arn, - logEventArn: updatedEsRule.rule.logEventArn, - type: 'kinesis', - value: kinesisArn2, - }, + queue_url: 'fake-queue-url', + workflow, } ); - t.deepEqual(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'kinesis', - value: kinesisArn2, - arn: updatedPgRule.arn, - log_event_arn: updatedPgRule.log_event_arn, - }); + + // Set required property to null to trigger create error + originalApiRule.workflow = null; + + const response = await request(app) + .put(`/rules/${originalApiRule.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(originalApiRule) + .expect(400); + const { message } = response.body; + t.true(message.includes('The record has validation errors. Rule workflow is undefined')); }); -test.serial('PUT creates the same SQS rule in PostgreSQL/Elasticsearch', async (t) => { +test('PUT returns a 400 response if record is missing type property', async (t) => { const { - pgProvider, - pgCollection, - } = t.context; - - const queue1 = randomId('queue'); - const queue2 = randomId('queue'); + originalApiRule, + originalPgRecord, + } = await createRuleTestRecords( + t.context, + { + queue_url: 'fake-queue-url', + workflow, + } + ); + originalApiRule.rule.type = null; + const response = await request(app) + .put(`/rules/${originalPgRecord.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(originalApiRule) + .expect(400); + const { message } = response.body; + t.true(message.includes('The record has validation errors. Rule type is undefined.')); +}); - const { queueUrl: queueUrl1 } = await createSqsQueues(queue1); - const { queueUrl: queueUrl2 } = await createSqsQueues(queue2, 4, '100'); +test('PUT returns a 400 response if rule name is invalid', async (t) => { + const { + originalApiRule, + originalPgRecord, + } = await createRuleTestRecords( + t.context, + { + queue_url: 'fake-queue-url', + workflow, + } + ); + originalApiRule.name = 'bad rule name'; + const response = await request(app) + .put(`/rules/${originalPgRecord.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(originalApiRule) + .expect(400); + const { message } = response.body; + t.true(message.includes(originalApiRule.name)); +}); +test('PUT returns a 400 response if rule type is invalid', async (t) => { const { originalApiRule, originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( + t.context, { - ...t.context, - }, - { + queue_url: 'fake-queue-url', workflow, - name: randomId('rule'), - state: 'ENABLED', - type: 'sqs', - value: queueUrl1, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, } ); + originalApiRule.rule.type = 'invalid'; - const expectedMeta = { - visibilityTimeout: 300, - retries: 3, - }; - - t.deepEqual(originalPgRecord.meta, expectedMeta); - t.deepEqual(originalEsRecord.meta, expectedMeta); - - const updateRule = { - ...originalApiRule, - rule: { - type: 'sqs', - value: queueUrl2, - }, - meta: { - retries: 2, - }, - }; - const expressRequest = { - params: { - name: originalApiRule.name, - }, - body: updateRule, - }; - const response = buildFakeExpressResponse(); - await put(expressRequest, response); + const response = await request(app) + .put(`/rules/${originalPgRecord.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(originalApiRule) + .expect(400); - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - updateRule.name - ); - const expectedMetaUpdate = { - visibilityTimeout: 100, - retries: 2, - }; - t.deepEqual( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sqs', - value: queueUrl2, - }, - meta: expectedMetaUpdate, - } - ); - t.deepEqual(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'sqs', - value: queueUrl2, - meta: expectedMetaUpdate, - }); + const { message } = response.body; + const regexp = new RegExp('The record has validation errors:.*rule.type.*should be equal to one of the allowed values'); + t.truthy(message.match(regexp)); }); -test.serial('PUT keeps initial trigger information if writing to PostgreSQL fails', async (t) => { - const { - pgProvider, - pgCollection, - } = t.context; - - const topic1 = await createSnsTopic(randomId('topic1_')); - const topic2 = await createSnsTopic(randomId('topic2_')); - +test('PUT returns a 400 response if rule value is not specified for non-onetime rule', async (t) => { const { originalApiRule, originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( + t.context, { - ...t.context, - }, - { + queue_url: 'fake-queue-url', workflow, - state: 'ENABLED', - type: 'sns', - value: topic1.TopicArn, - collection: { - name: pgCollection.name, - version: pgCollection.version, - }, - provider: pgProvider.name, } ); + originalApiRule.rule.type = 'kinesis'; - t.truthy(originalEsRecord.rule.value); - t.truthy(originalPgRecord.value); - - const updateRule = { - ...originalApiRule, - rule: { - type: 'sns', - value: topic2.TopicArn, - }, - }; - - const expressRequest = { - params: { - name: originalApiRule.name, - }, - body: updateRule, - testContext: { - rulePgModel: { - get: () => Promise.resolve(originalPgRecord), - upsert: () => { - throw new Error('PG fail'); - }, - }, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - put(expressRequest, response), - { message: 'PG fail' } - ); - - const updatedPgRule = await t.context.rulePgModel - .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name - ); - - t.is(updatedEsRule.rule.arn, originalEsRecord.rule.arn); - t.is(updatedPgRule.arn, originalPgRecord.arn); + const response = await request(app) + .put(`/rules/${originalPgRecord.name}`) + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .send(originalApiRule) + .expect(400); - t.like( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sns', - value: topic1.TopicArn, - }, - } - ); - t.like(updatedPgRule, { - ...originalPgRecord, - updated_at: updatedPgRule.updated_at, - type: 'sns', - value: topic1.TopicArn, - }); + const { message } = response.body; + const regexp = new RegExp('Rule value is undefined for kinesis rule'); + t.truthy(message.match(regexp)); }); -test.serial('PUT keeps initial trigger information if writing to Elasticsearch fails', async (t) => { +test.serial('PUT keeps initial trigger information if writing to PostgreSQL fails', async (t) => { const { pgProvider, pgCollection, @@ -2877,7 +1773,6 @@ test.serial('PUT keeps initial trigger information if writing to Elasticsearch f const { originalApiRule, originalPgRecord, - originalEsRecord, } = await createRuleTestRecords( { ...t.context, @@ -2895,7 +1790,6 @@ test.serial('PUT keeps initial trigger information if writing to Elasticsearch f } ); - t.truthy(originalEsRecord.rule.value); t.truthy(originalPgRecord.value); const updateRule = { @@ -2912,11 +1806,10 @@ test.serial('PUT keeps initial trigger information if writing to Elasticsearch f }, body: updateRule, testContext: { - esClient: { - client: { - index: () => { - throw new Error('ES fail'); - }, + rulePgModel: { + get: () => Promise.resolve(originalPgRecord), + upsert: () => { + throw new Error('PG fail'); }, }, }, @@ -2926,30 +1819,14 @@ test.serial('PUT keeps initial trigger information if writing to Elasticsearch f await t.throwsAsync( put(expressRequest, response), - { message: 'ES fail' } + { message: 'PG fail' } ); const updatedPgRule = await t.context.rulePgModel .get(t.context.testKnex, { name: updateRule.name }); - const updatedEsRule = await t.context.esRulesClient.get( - originalPgRecord.name - ); - t.is(updatedEsRule.rule.arn, originalEsRecord.rule.arn); t.is(updatedPgRule.arn, originalPgRecord.arn); - t.like( - updatedEsRule, - { - ...originalEsRecord, - updatedAt: updatedEsRule.updatedAt, - timestamp: updatedEsRule.timestamp, - rule: { - type: 'sns', - value: topic1.TopicArn, - }, - } - ); t.like(updatedPgRule, { ...originalPgRecord, updated_at: updatedPgRule.updated_at, @@ -3022,7 +1899,7 @@ test.serial('PATCH returns 200 for version value greater than the configured val t.is(response.status, 200); }); -test('DELETE returns a 404 if PostgreSQL and Elasticsearch rule cannot be found', async (t) => { +test('DELETE returns a 404 if rule cannot be found', async (t) => { const nonExistentRule = fakeRuleRecordFactory(); const response = await request(app) .delete(`/rules/${nonExistentRule.name}`) @@ -3032,74 +1909,6 @@ test('DELETE returns a 404 if PostgreSQL and Elasticsearch rule cannot be found' t.is(response.body.message, 'No record found'); }); -test('DELETE deletes rule that exists in PostgreSQL but not Elasticsearch', async (t) => { - const { - esRulesClient, - rulePgModel, - testKnex, - } = t.context; - const newRule = fakeRuleRecordFactory(); - delete newRule.collection; - delete newRule.provider; - await rulePgModel.create(testKnex, newRule); - - t.false( - await esRulesClient.exists( - newRule.name - ) - ); - t.true( - await rulePgModel.exists(testKnex, { - name: newRule.name, - }) - ); - const response = await request(app) - .delete(`/rules/${newRule.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - const { message } = response.body; - const dbRecords = await rulePgModel - .search(testKnex, { name: newRule.name }); - - t.is(dbRecords.length, 0); - t.is(message, 'Record deleted'); -}); - -test('DELETE deletes rule that exists in Elasticsearch but not PostgreSQL', async (t) => { - const { - esClient, - esIndex, - esRulesClient, - rulePgModel, - testKnex, - } = t.context; - const newRule = fakeRuleRecordFactory(); - await indexer.indexRule(esClient, newRule, esIndex); - - t.true( - await esRulesClient.exists( - newRule.name - ) - ); - t.false( - await rulePgModel.exists(testKnex, { - name: newRule.name, - }) - ); - const response = await request(app) - .delete(`/rules/${newRule.name}`) - .set('Accept', 'application/json') - .set('Authorization', `Bearer ${jwtAuthToken}`) - .expect(200); - const { message } = response.body; - const dbRecords = await rulePgModel - .search(t.context.testKnex, { name: newRule.name }); - - t.is(dbRecords.length, 0); - t.is(message, 'Record deleted'); -}); - test('DELETE deletes a rule', async (t) => { const { originalPgRecord, @@ -3123,103 +1932,4 @@ test('DELETE deletes a rule', async (t) => { t.is(dbRecords.length, 0); t.is(message, 'Record deleted'); - t.false( - await t.context.esRulesClient.exists( - originalPgRecord.name - ) - ); -}); - -test('del() does not remove from Elasticsearch if removing from PostgreSQL fails', async (t) => { - const { - originalPgRecord, - } = await createRuleTestRecords( - t.context, - { - workflow, - } - ); - - const fakeRulesPgModel = { - delete: () => { - throw new Error('something bad'); - }, - get: () => Promise.resolve(originalPgRecord), - }; - - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - testContext: { - knex: t.context.testKnex, - rulePgModel: fakeRulesPgModel, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - del(expressRequest, response), - { message: 'something bad' } - ); - - t.true( - await t.context.rulePgModel.exists(t.context.testKnex, { - name: originalPgRecord.name, - }) - ); - t.true( - await t.context.esRulesClient.exists( - originalPgRecord.name - ) - ); -}); - -test('del() does not remove from PostgreSQL if removing from Elasticsearch fails', async (t) => { - const { - originalPgRecord, - } = await createRuleTestRecords( - t.context, - { - workflow, - } - ); - - const fakeEsClient = { - client: { - delete: () => { - throw new Error('something bad'); - }, - }, - initializeEsClient: () => Promise.resolve(), - }; - - const expressRequest = { - params: { - name: originalPgRecord.name, - }, - testContext: { - knex: t.context.testKnex, - esClient: fakeEsClient, - }, - }; - - const response = buildFakeExpressResponse(); - - await t.throwsAsync( - del(expressRequest, response), - { message: 'something bad' } - ); - - t.true( - await t.context.rulePgModel.exists(t.context.testKnex, { - name: originalPgRecord.name, - }) - ); - t.true( - await t.context.esRulesClient.exists( - originalPgRecord.name - ) - ); }); diff --git a/packages/api/tests/endpoints/stats.js b/packages/api/tests/endpoints/test-stats.js similarity index 70% rename from packages/api/tests/endpoints/stats.js rename to packages/api/tests/endpoints/test-stats.js index 1853a7c53ac..f2168a34c81 100644 --- a/packages/api/tests/endpoints/stats.js +++ b/packages/api/tests/endpoints/test-stats.js @@ -9,12 +9,6 @@ const awsServices = require('@cumulus/aws-client/services'); const s3 = require('@cumulus/aws-client/S3'); const { randomId } = require('@cumulus/common/test-utils'); -const models = require('../../models'); -const { - createFakeJwtAuthToken, - setAuthorizedOAuthUsers, -} = require('../../lib/testUtils'); - const { destroyLocalTestDb, generateLocalTestDb, @@ -24,7 +18,15 @@ const { fakeGranuleRecordFactory, migrationDir, localStackConnectionEnv, -} = require('../../../db/dist'); + fakeReconciliationReportRecordFactory, + ReconciliationReportPgModel, +} = require('@cumulus/db'); + +const models = require('../../models'); +const { + createFakeJwtAuthToken, + setAuthorizedOAuthUsers, +} = require('../../lib/testUtils'); const testDbName = randomId('collection'); @@ -74,41 +76,39 @@ test.before(async (t) => { t.context.collectionPgModel = new CollectionPgModel(); t.context.granulePgModel = new GranulePgModel(); + t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); const statuses = ['queued', 'failed', 'completed', 'running']; const errors = [{ Error: 'UnknownError' }, { Error: 'CumulusMessageAdapterError' }, { Error: 'IngestFailure' }, { Error: 'CmrFailure' }, {}]; - const granules = []; - const collections = []; - - range(20).map((num) => ( - collections.push(fakeCollectionRecordFactory({ - name: `testCollection${num}`, - cumulus_id: num, - })) - )); - - range(100).map((num) => ( - granules.push(fakeGranuleRecordFactory({ - collection_cumulus_id: num % 20, - status: statuses[num % 4], - created_at: num === 99 - ? new Date() : (new Date(2018 + (num % 6), (num % 12), (num % 30))), - updated_at: num === 99 - ? new Date() : (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), - error: errors[num % 5], - duration: num + (num / 10), - })) - )); - - await t.context.collectionPgModel.insert( - t.context.knex, - collections - ); - - await t.context.granulePgModel.insert( - t.context.knex, - granules - ); + const reconReportTypes = ['Granule Inventory', 'Granule Not Found', 'Inventory', 'ORCA Backup']; + const reconReportStatuses = ['Generated', 'Pending', 'Failed']; + + const collections = range(20).map((num) => fakeCollectionRecordFactory({ + name: `testCollection${num}`, + cumulus_id: num, + })); + + const granules = range(100).map((num) => fakeGranuleRecordFactory({ + collection_cumulus_id: num % 20, + status: statuses[num % 4], + created_at: num === 99 + ? new Date() : (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: num === 99 + ? new Date() : (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + error: errors[num % 5], + duration: num + (num / 10), + })); + + const reconReports = range(24).map((num) => fakeReconciliationReportRecordFactory({ + type: reconReportTypes[num % 4], + status: reconReportStatuses[num % 3], + created_at: (new Date(2024 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2024 + (num % 6), (num % 12), ((num + 1) % 29))), + })); + + await t.context.collectionPgModel.insert(t.context.knex, collections); + await t.context.granulePgModel.insert(t.context.knex, granules); + await t.context.reconciliationReportPgModel.insert(t.context.knex, reconReports); }); test.after.always(async (t) => { @@ -187,6 +187,12 @@ test('getType gets correct type for providers', (t) => { t.is(type, 'provider'); }); +test('getType gets correct type for reconciliation reports', (t) => { + const type = getType({ params: { type: 'reconciliationReports' } }); + + t.is(type, 'reconciliationReport'); +}); + test('getType returns undefined if type is not supported', (t) => { const type = getType({ params: { type: 'provide' } }); @@ -237,7 +243,7 @@ test('GET /stats returns correct response with date params filters values correc t.is(response.body.granules.value, 17); }); -test('GET /stats/aggregate returns correct response', async (t) => { +test('GET /stats/aggregate with type `granules` returns correct response', async (t) => { const response = await request(app) .get('/stats/aggregate?type=granules') .set('Accept', 'application/json') @@ -254,7 +260,7 @@ test('GET /stats/aggregate returns correct response', async (t) => { t.deepEqual(response.body.count, expectedCount); }); -test('GET /stats/aggregate filters correctly by date', async (t) => { +test('GET /stats/aggregate with type `granules` filters correctly by date', async (t) => { const response = await request(app) .get(`/stats/aggregate?type=granules×tamp__from=${(new Date(2020, 11, 28)).getTime()}×tamp__to=${(new Date(2023, 8, 30)).getTime()}`) .set('Accept', 'application/json') @@ -270,3 +276,38 @@ test('GET /stats/aggregate filters correctly by date', async (t) => { t.is(response.body.meta.count, 40); t.deepEqual(response.body.count, expectedCount); }); + +test('GET /stats/aggregate with type `reconciliationReports` and field `type` returns the correct response', async (t) => { + const response = await request(app) + .get('/stats/aggregate?type=reconciliationReports&field=type') + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .expect(200); + + const expectedCount = [ + { key: 'Granule Inventory', count: 6 }, + { key: 'Granule Not Found', count: 6 }, + { key: 'Inventory', count: 6 }, + { key: 'ORCA Backup', count: 6 }, + ]; + + t.is(response.body.meta.count, 24); + t.deepEqual(response.body.count, expectedCount); +}); + +test('GET /stats/aggregate with type `reconciliationReports` and field `status` returns the correct response', async (t) => { + const response = await request(app) + .get('/stats/aggregate?type=reconciliationReports&field=status') + .set('Accept', 'application/json') + .set('Authorization', `Bearer ${jwtAuthToken}`) + .expect(200); + + const expectedCount = [ + { key: 'Failed', count: 8 }, + { key: 'Generated', count: 8 }, + { key: 'Pending', count: 8 }, + ]; + + t.is(response.body.meta.count, 24); + t.deepEqual(response.body.count, expectedCount); +}); diff --git a/packages/api/tests/helpers/create-test-data.js b/packages/api/tests/helpers/create-test-data.js index 1867f7a66a7..09697ce08e8 100644 --- a/packages/api/tests/helpers/create-test-data.js +++ b/packages/api/tests/helpers/create-test-data.js @@ -47,8 +47,9 @@ const metadataFileFixture = fs.readFileSync(path.resolve(__dirname, '../data/met * @param {Knex} params.dbClient - Knex client * @param {number} params.executionCumulusId - executionId for execution record to link * @param {number} params.collectionId - collectionId for the granule's parent collection - * @param {number} params.collectionCumulusId - cumulus_id for the granule's parent collection * @param {boolean} params.published - if the granule should be marked published to CMR + * @param {Object} [params.granuleParams] - additional granule parameters + * @param {number} [params.collectionCumulusId] - cumulus_id for the granule's parent collection * @returns {Object} fake granule object */ async function createGranuleAndFiles({ diff --git a/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-index.js b/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-index.js index dfa6d152e0b..15fafbe37ad 100644 --- a/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-index.js +++ b/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-index.js @@ -29,13 +29,6 @@ const { const { UnmetRequirementsError, } = require('@cumulus/errors'); -const { - Search, -} = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { constructCollectionId, } = require('@cumulus/message/Collections'); @@ -140,26 +133,6 @@ test.before(async (t) => { t.context.testKnex = knex; t.context.testKnexAdmin = knexAdmin; - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - - t.context.esExecutionsClient = new Search( - {}, - 'execution', - t.context.esIndex - ); - t.context.esPdrsClient = new Search( - {}, - 'pdr', - t.context.esIndex - ); - t.context.esGranulesClient = new Search( - {}, - 'granule', - t.context.esIndex - ); - t.context.collectionPgModel = new CollectionPgModel(); t.context.executionPgModel = new ExecutionPgModel(); t.context.granulePgModel = new GranulePgModel(); @@ -283,7 +256,6 @@ test.after.always(async (t) => { knexAdmin: t.context.testKnexAdmin, testDbName: t.context.testDbName, }); - await cleanupTestIndex(t.context); await sns().send(new DeleteTopicCommand({ TopicArn: ExecutionsTopicArn })); await sns().send(new DeleteTopicCommand({ TopicArn: PdrsTopicArn })); }); diff --git a/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-write-pdr.js b/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-write-pdr.js index 17c873f9fb7..bc1c84b6465 100644 --- a/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-write-pdr.js +++ b/packages/api/tests/lambdas/sf-event-sqs-to-db-records/test-write-pdr.js @@ -17,7 +17,6 @@ const { translatePostgresPdrToApiPdr, migrationDir, } = require('@cumulus/db'); -const { Search } = require('@cumulus/es-client/search'); const { createSnsTopic } = require('@cumulus/aws-client/SNS'); const { sns, sqs } = require('@cumulus/aws-client/services'); const { @@ -25,10 +24,6 @@ const { DeleteTopicCommand, } = require('@aws-sdk/client-sns'); const { ReceiveMessageCommand } = require('@aws-sdk/client-sqs'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { generatePdrRecord, @@ -45,15 +40,6 @@ test.before(async (t) => { ); t.context.knexAdmin = knexAdmin; t.context.knex = knex; - - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esPdrClient = new Search( - {}, - 'pdr', - t.context.esIndex - ); }); test.beforeEach(async (t) => { @@ -166,7 +152,6 @@ test.after.always(async (t) => { await destroyLocalTestDb({ ...t.context, }); - await cleanupTestIndex(t.context); }); test('generatePdrRecord() generates correct PDR record', (t) => { @@ -320,7 +305,6 @@ test.serial('writePdr() does not update PDR record if update is from an older ex }); const pgRecord = await pdrPgModel.get(knex, { name: pdr.name }); - const esRecord = await t.context.esPdrClient.get(pdr.name); const stats = { processing: 0, @@ -330,10 +314,6 @@ test.serial('writePdr() does not update PDR record if update is from an older ex status: 'completed', stats, }); - t.like(esRecord, { - status: 'completed', - stats, - }); cumulusMessage.meta.status = 'running'; cumulusMessage.payload.running = ['arn2']; @@ -349,18 +329,13 @@ test.serial('writePdr() does not update PDR record if update is from an older ex }); const updatedPgRecord = await pdrPgModel.get(knex, { name: pdr.name }); - const updatedEsRecord = await t.context.esPdrClient.get(pdr.name); t.like(updatedPgRecord, { status: 'completed', stats, }); - t.like(updatedEsRecord, { - status: 'completed', - stats, - }); }); -test.serial('writePdr() saves a PDR record to PostgreSQL/Elasticsearch if PostgreSQL write is enabled', async (t) => { +test.serial('writePdr() saves a PDR record to PostgreSQL', async (t) => { const { cumulusMessage, knex, @@ -380,35 +355,9 @@ test.serial('writePdr() saves a PDR record to PostgreSQL/Elasticsearch if Postgr }); t.true(await pdrPgModel.exists(knex, { name: pdr.name })); - t.true(await t.context.esPdrClient.exists(pdr.name)); }); -test.serial('writePdr() saves a PDR record to PostgreSQL/Elasticsearch with same timestamps', async (t) => { - const { - cumulusMessage, - knex, - collectionCumulusId, - providerCumulusId, - executionCumulusId, - pdr, - pdrPgModel, - } = t.context; - - await writePdr({ - cumulusMessage, - collectionCumulusId, - providerCumulusId, - executionCumulusId: executionCumulusId, - knex, - }); - - const pgRecord = await pdrPgModel.get(knex, { name: pdr.name }); - const esRecord = await t.context.esPdrClient.get(pdr.name); - t.is(pgRecord.created_at.getTime(), esRecord.createdAt); - t.is(pgRecord.updated_at.getTime(), esRecord.updatedAt); -}); - -test.serial('writePdr() does not write to PostgreSQL/Elasticsearch if PostgreSQL write fails', async (t) => { +test.serial('writePdr() does not write to PostgreSQL if PostgreSQL write fails', async (t) => { const { cumulusMessage, knex, @@ -450,51 +399,6 @@ test.serial('writePdr() does not write to PostgreSQL/Elasticsearch if PostgreSQL ); t.false(await pdrPgModel.exists(knex, { name: pdr.name })); - t.false(await t.context.esPdrClient.exists(pdr.name)); -}); - -test.serial('writePdr() does not write to PostgreSQL/Elasticsearch if Elasticsearch write fails', async (t) => { - const { - cumulusMessage, - knex, - collectionCumulusId, - providerCumulusId, - pdrPgModel, - } = t.context; - - const pdr = { - name: cryptoRandomString({ length: 5 }), - PANSent: false, - PANmessage: 'test', - }; - cumulusMessage.payload = { - pdr, - }; - - cumulusMessage.meta.status = 'completed'; - - const fakeEsClient = { - initializeEsClient: () => Promise.resolve(), - client: { - update: () => { - throw new Error('PDR ES error'); - }, - }, - }; - - await t.throwsAsync( - writePdr({ - cumulusMessage, - collectionCumulusId, - providerCumulusId, - knex, - esClient: fakeEsClient, - }), - { message: 'PDR ES error' } - ); - - t.false(await pdrPgModel.exists(knex, { name: pdr.name })); - t.false(await t.context.esPdrClient.exists(pdr.name)); }); test.serial('writePdr() successfully publishes an SNS message', async (t) => { diff --git a/packages/api/tests/lambdas/test-bootstrap.js b/packages/api/tests/lambdas/test-bootstrap.js deleted file mode 100644 index c0797c6496b..00000000000 --- a/packages/api/tests/lambdas/test-bootstrap.js +++ /dev/null @@ -1,43 +0,0 @@ -const test = require('ava'); -const sinon = require('sinon'); - -const { handler } = require('../../lambdas/bootstrap'); - -test('handler calls bootstrapFunction with expected values', async (t) => { - const bootstrapFunctionStub = sinon.stub(); - const testContext = { - bootstrapFunction: bootstrapFunctionStub, - }; - - const hostName = 'fakehost'; - - const actual = await handler({ - testContext, - removeAliasConflict: true, - elasticsearchHostname: hostName, - }); - - t.deepEqual(actual, { Data: {}, Status: 'SUCCESS' }); - t.true(bootstrapFunctionStub.calledWith({ - host: hostName, - removeAliasConflict: true, - })); -}); - -test('handler throws with error/status on bootstrap function failure', async (t) => { - const errorMessage = 'Fake Error'; - const bootstrapFunctionStub = () => { - throw new Error(errorMessage); - }; - const testContext = { - bootstrapFunction: bootstrapFunctionStub, - }; - - const hostName = 'fakehost'; - - await t.throwsAsync(handler({ - testContext, - removeAliasConflict: true, - elasticsearchHostname: hostName, - }), { message: errorMessage }); -}); diff --git a/packages/api/tests/lambdas/test-bulk-granule-delete.js b/packages/api/tests/lambdas/test-bulk-granule-delete.js index c1f16faaad8..e42afe304dd 100644 --- a/packages/api/tests/lambdas/test-bulk-granule-delete.js +++ b/packages/api/tests/lambdas/test-bulk-granule-delete.js @@ -13,11 +13,7 @@ const { } = require('@cumulus/db'); const { createBucket, deleteS3Buckets } = require('@cumulus/aws-client/S3'); const { randomId, randomString } = require('@cumulus/common/test-utils'); -const { Search } = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); + const { sns, sqs } = require('@cumulus/aws-client/services'); const { SubscribeCommand, @@ -44,11 +40,6 @@ test.before(async (t) => { const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); t.context.knex = knex; t.context.knexAdmin = knexAdmin; - - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esGranulesClient = new Search({}, 'granule', t.context.esIndex); }); test.beforeEach(async (t) => { @@ -87,7 +78,6 @@ test.after.always(async (t) => { knexAdmin: t.context.knexAdmin, testDbName, }); - await cleanupTestIndex(t.context); }); test('bulkGranuleDelete does not fail on published granules if payload.forceRemoveFromCmr is true', async (t) => { @@ -164,17 +154,6 @@ test('bulkGranuleDelete does not fail on published granules if payload.forceRemo { granule_id: pgGranuleId2, collection_cumulus_id: pgCollectionCumulusId2 } )); - t.false( - await t.context.esGranulesClient.exists( - pgGranuleId1 - ) - ); - t.false( - await t.context.esGranulesClient.exists( - pgGranuleId2 - ) - ); - const s3Buckets = granules[0].s3Buckets; t.teardown(() => deleteS3Buckets([ s3Buckets.protected.name, diff --git a/packages/api/tests/lambdas/test-bulk-operation.js b/packages/api/tests/lambdas/test-bulk-operation.js index 3b28d46c994..f858da57e74 100644 --- a/packages/api/tests/lambdas/test-bulk-operation.js +++ b/packages/api/tests/lambdas/test-bulk-operation.js @@ -49,7 +49,6 @@ const esSearchStub = sandbox.stub(); const esScrollStub = sandbox.stub(); FakeEsClient.prototype.scroll = esScrollStub; FakeEsClient.prototype.search = esSearchStub; - const bulkOperation = proxyquire('../../lambdas/bulk-operation', { '../lib/granules': proxyquire('../../lib/granules', { '@cumulus/es-client/search': { @@ -392,6 +391,7 @@ test.serial('bulk operation BULK_GRANULE applies workflow to granules returned b }); await verifyGranulesQueuedStatus(t); }); + test.serial('applyWorkflowToGranules sets the granules status to queued', async (t) => { await setUpExistingDatabaseRecords(t); const workflowName = 'test-workflow'; diff --git a/packages/api/tests/lambdas/test-cleanExecutions.js b/packages/api/tests/lambdas/test-cleanExecutions.js deleted file mode 100644 index e19e251b058..00000000000 --- a/packages/api/tests/lambdas/test-cleanExecutions.js +++ /dev/null @@ -1,553 +0,0 @@ -/* eslint-disable no-await-in-loop */ -const test = require('ava'); -const moment = require('moment'); -const clone = require('lodash/clone'); -const { - translatePostgresExecutionToApiExecution, - fakeExecutionRecordFactory, - localStackConnectionEnv, -} = require('@cumulus/db'); -const { cleanupTestIndex, createTestIndex } = require('@cumulus/es-client/testUtils'); -const { handler, getExpirationDate, cleanupExpiredESExecutionPayloads } = require('../../lambdas/cleanExecutions'); -test.beforeEach(async (t) => { - const { esIndex, esClient, searchClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.searchClient = searchClient; - - const records = []; - for (let i = 0; i < 20; i += 2) { - records.push(await translatePostgresExecutionToApiExecution(fakeExecutionRecordFactory({ - updated_at: moment().subtract(i, 'days').toDate(), - final_payload: '{"a": "b"}', - original_payload: '{"b": "c"}', - status: 'completed', - cumulus_id: i, - }))); - records.push(await translatePostgresExecutionToApiExecution(fakeExecutionRecordFactory({ - updated_at: moment().subtract(i, 'days').toDate(), - final_payload: '{"a": "b"}', - original_payload: '{"b": "c"}', - status: 'running', - cumulus_id: i + 1, - }))); - } - for (const record of records) { - await t.context.esClient.client.index({ - body: record, - id: record.cumulusId, - index: t.context.esIndex, - type: 'execution', - refresh: true, - }); - } -}); - -test.afterEach.always(async (t) => { - await cleanupTestIndex(t.context); -}); - -const esPayloadsEmpty = (entry) => !entry.finalPayload && !entry.orginalPayload; - -test.serial('handler() handles running expiration', async (t) => { - const env = clone(process.env); - process.env = localStackConnectionEnv; - process.env.PG_DATABASE = t.context.testDbName; - process.env.ES_INDEX = t.context.esIndex; - process.env.LOCAL_ES_HOST = 'localhost'; - let expirationDays = 4; - let expirationDate = getExpirationDate(expirationDays); - process.env.CLEANUP_NON_RUNNING = 'false'; - process.env.CLEANUP_RUNNING = 'true'; - process.env.PAYLOAD_TIMEOUT = expirationDays; - - await handler(); - - let massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - massagedEsExecutions.results.forEach((massagedExecution) => { - if (massagedExecution.updatedAt <= expirationDate && massagedExecution.status === 'running') { - t.true(esPayloadsEmpty(massagedExecution)); - } else { - t.false(esPayloadsEmpty(massagedExecution)); - } - }); - - expirationDays = 2; - expirationDate = getExpirationDate(expirationDays); - process.env.PAYLOAD_TIMEOUT = expirationDays; - - await handler(); - - massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - massagedEsExecutions.results.forEach((massagedExecution) => { - if (massagedExecution.updatedAt <= expirationDate.getTime() && massagedExecution.status === 'running') { - t.true(esPayloadsEmpty(massagedExecution)); - } else { - t.false(esPayloadsEmpty(massagedExecution)); - } - }); - process.env = env; -}); - -test.serial('handler() handles non running expiration', async (t) => { - const env = clone(process.env); - process.env = localStackConnectionEnv; - process.env.PG_DATABASE = t.context.testDbName; - process.env.ES_INDEX = t.context.esIndex; - let expirationDays = 5; - let expirationDate = getExpirationDate(expirationDays); - process.env.CLEANUP_NON_RUNNING = 'true'; - process.env.CLEANUP_RUNNING = 'false'; - process.env.PAYLOAD_TIMEOUT = expirationDays; - await handler(); - - let massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - - massagedEsExecutions.results.forEach((massagedExecution) => { - if (massagedExecution.updatedAt <= expirationDate && massagedExecution.status !== 'running') { - t.true(esPayloadsEmpty(massagedExecution)); - } else { - t.false(esPayloadsEmpty(massagedExecution)); - } - }); - - expirationDays = 3; - expirationDate = getExpirationDate(expirationDays); - process.env.PAYLOAD_TIMEOUT = expirationDays; - - await handler(); - - massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - massagedEsExecutions.results.forEach((massagedExecution) => { - if (massagedExecution.updatedAt <= expirationDate.getTime() && massagedExecution.status !== 'running') { - t.true(esPayloadsEmpty(massagedExecution)); - } else { - t.false(esPayloadsEmpty(massagedExecution)); - } - }); - process.env = env; -}); - -test.serial('handler() handles both expirations', async (t) => { - const env = clone(process.env); - process.env = localStackConnectionEnv; - process.env.PG_DATABASE = t.context.testDbName; - process.env.ES_INDEX = t.context.esIndex; - process.env.LOCAL_ES_HOST = 'localhost'; - let payloadTimeout = 9; - let payloadExpiration = getExpirationDate(payloadTimeout); - - process.env.CLEANUP_RUNNING = 'true'; - process.env.CLEANUP_NON_RUNNING = 'true'; - process.env.PAYLOAD_TIMEOUT = payloadTimeout; - - await handler(); - - let massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - massagedEsExecutions.results.forEach((massagedExecution) => { - if (massagedExecution.updatedAt <= payloadExpiration.getTime()) { - t.true(esPayloadsEmpty(massagedExecution)); - } else { - t.false(esPayloadsEmpty(massagedExecution)); - } - }); - payloadTimeout = 8; - - payloadExpiration = getExpirationDate(payloadTimeout); - process.env.PAYLOAD_TIMEOUT = payloadTimeout; - - await handler(); - - massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - massagedEsExecutions.results.forEach((massagedExecution) => { - if (massagedExecution.updatedAt <= payloadExpiration.getTime()) { - t.true(esPayloadsEmpty(massagedExecution)); - } else { - t.false(esPayloadsEmpty(massagedExecution)); - } - }); - process.env = env; -}); - -test.serial('handler() throws errors when misconfigured', async (t) => { - const env = clone(process.env); - process.env.CLEANUP_RUNNING = 'false'; - process.env.CLEANUP_NON_RUNNING = 'false'; - - await t.throwsAsync(handler(), { - message: 'running and non-running executions configured to be skipped, nothing to do', - }); - - process.env.CLEANUP_RUNNING = 'false'; - process.env.CLEANUP_NON_RUNNING = 'true'; - process.env.PAYLOAD_TIMEOUT = 'frogs'; - await t.throwsAsync(handler(), { - message: 'Invalid number of days specified in configuration for payloadTimeout: frogs', - }); - process.env = env; -}); - -test.serial('handler() iterates through data in batches when updateLimit is set low', async (t) => { - const env = clone(process.env); - - process.env = localStackConnectionEnv; - process.env.PG_DATABASE = t.context.testDbName; - process.env.ES_INDEX = t.context.esIndex; - process.env.LOCAL_ES_HOST = 'localhost'; - - process.env.CLEANUP_RUNNING = 'true'; - process.env.CLEANUP_NON_RUNNING = 'true'; - process.env.PAYLOAD_TIMEOUT = 2; - - process.env.UPDATE_LIMIT = 2; - - await handler(); - - let massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - let esCleanedCount = 0; - massagedEsExecutions.results.forEach((massagedExecution) => { - if (esPayloadsEmpty(massagedExecution)) esCleanedCount += 1; - }); - t.is(esCleanedCount, 2); - - await handler(); - - massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - esCleanedCount = 0; - massagedEsExecutions.results.forEach((massagedExecution) => { - if (esPayloadsEmpty(massagedExecution)) esCleanedCount += 1; - }); - t.is(esCleanedCount, 4); - - process.env.UPDATE_LIMIT = 12; - - await handler(); - - massagedEsExecutions = await t.context.searchClient.query({ - index: t.context.esIndex, - type: 'execution', - body: {}, - size: 30, - }); - esCleanedCount = 0; - massagedEsExecutions.results.forEach((massagedExecution) => { - if (esPayloadsEmpty(massagedExecution)) esCleanedCount += 1; - }); - t.is(esCleanedCount, 16); - - process.env = env; -}); - -test('cleanupExpiredEsExecutionPayloads() for just running removes expired running executions', async (t) => { - let timeoutDays = 6; - await cleanupExpiredESExecutionPayloads( - timeoutDays, - true, - false, - 100, - t.context.esIndex - ); - // await es refresh - - let expiration = moment().subtract(timeoutDays, 'days').toDate().getTime(); - let relevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - lte: expiration, - }, - }, - }, - }, - } - ); - for (const execution of relevantExecutions.results) { - if (execution.status === 'running') { - t.true(execution.finalPayload === undefined); - t.true(execution.originalPayload === undefined); - } else { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } - } - let irrelevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - gt: expiration, - }, - }, - }, - }, - } - ); - for (const execution of irrelevantExecutions.results) { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } - - timeoutDays = 2; - await cleanupExpiredESExecutionPayloads( - timeoutDays, - true, - false, - 100, - t.context.esIndex - ); - - expiration = moment().subtract(timeoutDays, 'days').toDate().getTime(); - relevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - lte: expiration, - }, - }, - }, - }, - } - ); - for (const execution of relevantExecutions.results) { - if (execution.status === 'running') { - t.true(execution.finalPayload === undefined); - t.true(execution.originalPayload === undefined); - } else { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } - } - irrelevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - gt: expiration, - }, - }, - }, - }, - } - ); - for (const execution of irrelevantExecutions.results) { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } -}); - -test('cleanupExpiredEsExecutionPayloads() for just nonRunning removes expired non running executions', async (t) => { - let timeoutDays = 6; - await cleanupExpiredESExecutionPayloads( - timeoutDays, - false, - true, - 100, - t.context.esIndex - ); - - let expiration = moment().subtract(timeoutDays, 'days').toDate().getTime(); - - let relevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - lte: expiration, - }, - }, - }, - }, - } - ); - for (const execution of relevantExecutions.results) { - if (execution.status !== 'running') { - t.true(execution.finalPayload === undefined); - t.true(execution.originalPayload === undefined); - } else { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } - } - let irrelevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - gt: expiration, - }, - }, - }, - }, - } - ); - for (const execution of irrelevantExecutions.results) { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } - - timeoutDays = 2; - await cleanupExpiredESExecutionPayloads( - timeoutDays, - false, - true, - 100, - t.context.esIndex - ); - - expiration = moment().subtract(timeoutDays, 'days').toDate().getTime(); - relevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - lte: expiration, - }, - }, - }, - }, - } - ); - for (const execution of relevantExecutions.results) { - if (execution.status !== 'running') { - t.true(execution.finalPayload === undefined); - t.true(execution.originalPayload === undefined); - } else { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } - } - irrelevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - gt: expiration, - }, - }, - }, - }, - } - ); - for (const execution of irrelevantExecutions.results) { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } -}); - -test('cleanupExpiredEsExecutionPayloads() for running and nonRunning executions', async (t) => { - const timeoutDays = 5; - await cleanupExpiredESExecutionPayloads( - timeoutDays, - true, - true, - 100, - t.context.esIndex - ); - - const expiration = moment().subtract(timeoutDays, 'days').toDate().getTime(); - - const relevant = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - lte: expiration, - }, - }, - }, - }, - } - ); - for (const execution of relevant.results) { - t.true(execution.finalPayload === undefined); - t.true(execution.originalPayload === undefined); - } - const irrelevantExecutions = await t.context.searchClient.query( - { - index: t.context.esIndex, - type: 'execution', - body: { - query: { - range: { - updatedAt: { - gt: expiration, - }, - }, - }, - }, - } - ); - for (const execution of irrelevantExecutions.results) { - t.false(execution.finalPayload === undefined); - t.false(execution.originalPayload === undefined); - } -}); diff --git a/packages/api/tests/lambdas/test-create-reconciliation-report-internals.js b/packages/api/tests/lambdas/test-create-reconciliation-report-internals.js index 819a1acc326..ab4e9b248e9 100644 --- a/packages/api/tests/lambdas/test-create-reconciliation-report-internals.js +++ b/packages/api/tests/lambdas/test-create-reconciliation-report-internals.js @@ -9,7 +9,6 @@ const CRP = rewire('../../lambdas/create-reconciliation-report'); const linkingFilesToGranules = CRP.__get__('linkingFilesToGranules'); const isOneWayCollectionReport = CRP.__get__('isOneWayCollectionReport'); const isOneWayGranuleReport = CRP.__get__('isOneWayGranuleReport'); -const shouldAggregateGranulesForCollections = CRP.__get__('shouldAggregateGranulesForCollections'); test( 'isOneWayCollectionReport returns true only when one or more specific parameters ' @@ -86,39 +85,6 @@ test( } ); -test( - 'shouldAggregateGranulesForCollections returns true only when one or more specific parameters ' - + ' are present on the reconciliation report object.', - (t) => { - const paramsThatShouldReturnTrue = ['updatedAt__to', 'updatedAt__from']; - const paramsThatShouldReturnFalse = [ - 'stackName', - 'systemBucket', - 'startTimestamp', - 'anythingAtAll', - ]; - - paramsThatShouldReturnTrue.map((p) => - t.true(shouldAggregateGranulesForCollections({ [p]: randomId('value') }))); - - paramsThatShouldReturnFalse.map((p) => - t.false(shouldAggregateGranulesForCollections({ [p]: randomId('value') }))); - - const allTrueKeys = paramsThatShouldReturnTrue.reduce( - (accum, current) => ({ ...accum, [current]: randomId('value') }), - {} - ); - t.true(shouldAggregateGranulesForCollections(allTrueKeys)); - - const allFalseKeys = paramsThatShouldReturnFalse.reduce( - (accum, current) => ({ ...accum, [current]: randomId('value') }), - {} - ); - t.false(shouldAggregateGranulesForCollections(allFalseKeys)); - t.true(shouldAggregateGranulesForCollections({ ...allTrueKeys, ...allFalseKeys })); - } -); - test('linkingFilesToGranules return values', (t) => { const reportTypesToReturnFalse = ['Granule Inventory', 'Internal', 'Inventory']; const reportTypesToReturnTrue = ['Granule Not Found']; diff --git a/packages/api/tests/lambdas/test-create-reconciliation-report.js b/packages/api/tests/lambdas/test-create-reconciliation-report.js index 314d6db87a3..25b988d21bd 100644 --- a/packages/api/tests/lambdas/test-create-reconciliation-report.js +++ b/packages/api/tests/lambdas/test-create-reconciliation-report.js @@ -8,6 +8,7 @@ const pMap = require('p-map'); const omit = require('lodash/omit'); const range = require('lodash/range'); const sample = require('lodash/sample'); +const compact = require('lodash/compact'); const sinon = require('sinon'); const sortBy = require('lodash/sortBy'); const test = require('ava'); @@ -27,45 +28,45 @@ const { randomString, randomId } = require('@cumulus/common/test-utils'); const { CollectionPgModel, destroyLocalTestDb, - generateLocalTestDb, - localStackConnectionEnv, - FilePgModel, - GranulePgModel, + ExecutionPgModel, fakeCollectionRecordFactory, + fakeExecutionRecordFactory, fakeGranuleRecordFactory, + fakeProviderRecordFactory, + FilePgModel, + generateLocalTestDb, + GranulePgModel, + localStackConnectionEnv, migrationDir, + ProviderPgModel, + ReconciliationReportPgModel, + translateApiCollectionToPostgresCollection, + translateApiFiletoPostgresFile, translateApiGranuleToPostgresGranule, - translatePostgresCollectionToApiCollection, - ExecutionPgModel, - fakeExecutionRecordFactory, - upsertGranuleWithExecutionJoinRecord, + translatePostgresReconReportToApiReconReport, } = require('@cumulus/db'); const { getDistributionBucketMapKey } = require('@cumulus/distribution-utils'); -const indexer = require('@cumulus/es-client/indexer'); -const { Search, getEsClient } = require('@cumulus/es-client/search'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); const { - fakeCollectionFactory, fakeGranuleFactoryV2, fakeOrcaGranuleFactory, } = require('../../lib/testUtils'); const { handler: unwrappedHandler, reconciliationReportForGranules, reconciliationReportForGranuleFiles, } = require('../../lambdas/create-reconciliation-report'); -const models = require('../../models'); const { normalizeEvent } = require('../../lib/reconciliationReport/normalizeEvent'); const ORCASearchCatalogQueue = require('../../lib/ORCASearchCatalogQueue'); // Call normalize event on all input events before calling the handler. const handler = (event) => unwrappedHandler(normalizeEvent(event)); -let esAlias; -let esIndex; -let esClient; - const createBucket = (Bucket) => awsServices.s3().createBucket({ Bucket }); -const testDbName = `create_rec_reports_${cryptoRandomString({ length: 10 })}`; +const requiredStaticCollectionFields = { + granuleIdExtraction: randomString(), + granuleId: randomString(), + sampleFileName: randomString(), + files: [], +}; function createDistributionBucketMapFromBuckets(buckets) { let bucketMap = {}; @@ -124,59 +125,112 @@ async function storeFilesToS3(files) { ); } -/** - * Index a single collection to elasticsearch. If the collection object has an - * updatedAt value, use a sinon stub to set the time of the granule to that - * input time. - * @param {Object} collection - a collection object -* @returns {Promise} - promise of indexed collection with active granule -*/ -async function storeCollection(collection) { - let stub; - if (collection.updatedAt) { - stub = sinon.stub(Date, 'now').returns(collection.updatedAt); - } - try { - await indexer.indexCollection(esClient, collection, esAlias); - return indexer.indexGranule( - esClient, - fakeGranuleFactoryV2({ - collectionId: constructCollectionId(collection.name, collection.version), - updatedAt: collection.updatedAt, - provider: randomString(), - }), - esAlias - ); - } finally { - if (collection.updatedAt) stub.restore(); - } +async function storeCollectionAndGranuleToPostgres(collection, context) { + const postgresCollection = translateApiCollectionToPostgresCollection({ + ...collection, + ...requiredStaticCollectionFields, + }); + const [pgCollectionRecord] = await context.collectionPgModel.create( + context.knex, + postgresCollection + ); + const [pgProviderRecord] = await context.providerPgModel.create( + context.knex, + fakeProviderRecordFactory(), + ['name', 'cumulus_id'] + ); + const collectionGranule = fakeGranuleRecordFactory({ + updated_at: pgCollectionRecord.updated_at, + created_at: pgCollectionRecord.created_at, + collection_cumulus_id: pgCollectionRecord.cumulus_id, + provider_cumulus_id: pgProviderRecord.cumulus_id, + }); + await context.granulePgModel.create(context.knex, collectionGranule); + return { + granule: { + ...collectionGranule, + collectionId: `${collection.name}___${collection.version}`, + }, + collection: { + ...pgCollectionRecord, + providerName: pgProviderRecord.name, + }, + }; } -/** - * Index Dated collections to ES for testing timeranges. These need to happen - * in sequence because of the way we are stubbing Date.now() during indexing. - * - * @param {Array} collections - list of collection objects - * @returns {Promise} - Promise of collections indexed - */ -function storeCollectionsToElasticsearch(collections) { - let result = Promise.resolve(); - collections.forEach((collection) => { - result = result.then(() => storeCollection(collection)); - }); - return result; +async function storeCollectionsWithGranuleToPostgres(collections, context) { + const records = await Promise.all( + collections.map((collection) => storeCollectionAndGranuleToPostgres(collection, context)) + ); + return { + collections: records.map((record) => record.collection), + granules: records.map((record) => record.granule), + }; } -/** - * Index granules to ES for testing - * - * @param {Array} granules - list of granules objects - * @returns {Promise} - Promise of indexed granules - */ -async function storeGranulesToElasticsearch(granules) { - await Promise.all( - granules.map((granule) => indexer.indexGranule(esClient, granule, esAlias)) +async function generateRandomGranules(t, { + bucketRange = 2, + collectionRange = 10, + granuleRange = 10, + fileRange = 10, + stubCmr = true, +} = {}) { + const { filePgModel, granulePgModel, knex } = t.context; + + const dataBuckets = range(bucketRange).map(() => randomId('bucket')); + await Promise.all(dataBuckets.map((bucket) => + createBucket(bucket) + .then(() => t.context.bucketsToCleanup.push(bucket)))); + + // Write the buckets config to S3 + await storeBucketsConfigToS3( + dataBuckets, + t.context.systemBucket, + t.context.stackName ); + + // Create collections that are in sync + const matchingColls = range(collectionRange).map(() => ({ + name: randomId('name'), + version: randomId('vers'), + })); + const { collections: postgresCollections } = + await storeCollectionsWithGranuleToPostgres(matchingColls, t.context); + const collectionCumulusId = postgresCollections[0].cumulus_id; + + // Create random files + const pgGranules = await granulePgModel.insert( + knex, + range(granuleRange).map(() => fakeGranuleRecordFactory({ + collection_cumulus_id: collectionCumulusId, + })), + ['cumulus_id', 'granule_id'] + ); + const files = range(fileRange).map((i) => ({ + bucket: dataBuckets[i % dataBuckets.length], + key: randomId('key', 10), + granule_cumulus_id: pgGranules[i].cumulus_id, + })); + + // Store the files to S3 and postgres + await Promise.all([ + storeFilesToS3(files), + filePgModel.insert(knex, files), + ]); + + if (stubCmr) { + const cmrCollections = sortBy(matchingColls, ['name', 'version']) + .map((cmrCollection) => ({ + umm: { ShortName: cmrCollection.name, Version: cmrCollection.version }, + })); + CMR.prototype.searchConcept.restore(); + const cmrSearchStub = sinon.stub(CMR.prototype, 'searchConcept'); + cmrSearchStub.withArgs('collections').onCall(0).resolves(cmrCollections); + cmrSearchStub.withArgs('collections').onCall(1).resolves([]); + cmrSearchStub.withArgs('granules').resolves([]); + } + + return { files, granules: pgGranules, matchingColls, dataBuckets }; } async function fetchCompletedReport(reportRecord) { @@ -190,46 +244,20 @@ async function fetchCompletedReportString(reportRecord) { .then((response) => getObjectStreamContents(response.Body)); } -/** - * Looks up and returns the granulesIds given a list of collectionIds. - * @param {Array} collectionIds - list of collectionIds - * @returns {Array} list of matching granuleIds - */ -async function granuleIdsFromCollectionIds(collectionIds) { - const esValues = await (new Search( - { queryStringParameters: { collectionId__in: collectionIds.join(',') } }, - 'granule', - esAlias - )).query(); - return esValues.results.map((value) => value.granuleId); -} - -/** - * Looks up and returns the providers given a list of collectionIds. - * @param {Array} collectionIds - list of collectionIds - * @returns {Array} list of matching providers - */ -async function providersFromCollectionIds(collectionIds) { - const esValues = await (new Search( - { queryStringParameters: { collectionId__in: collectionIds.join(',') } }, - 'granule', - esAlias - )).query(); - - return esValues.results.map((value) => value.provider); -} - const randomBetween = (a, b) => Math.floor(Math.random() * (b - a + 1) + a); const randomTimeBetween = (t1, t2) => randomBetween(t1, t2); /** - * Prepares localstack with a number of active granules. Sets up ES with + * Prepares localstack with a number of active granules. Sets up pg with * random collections where some fall within the start and end timestamps. - * Also creates a number that are only in ES, as well as some that are only + * Also creates a number that are only in pg, as well as some that are only * "returned by CMR" (as a stubbed function) - * @param {Object} t - AVA test context. - * @returns {Object} setupVars - Object with information about the current - * state of elasticsearch and CMR mock. + * + * @param t.t + * @param {object} t - AVA test context. + * @param t.params + * @returns {object} setupVars - Object with information about the current + * state of pg and CMR mock. * The object returned has: * + startTimestamp - beginning of matching timerange * + endTimestamp - end of matching timerange @@ -237,13 +265,13 @@ const randomTimeBetween = (t1, t2) => randomBetween(t1, t2); * timestamps and included in the CMR mock * + matchingCollectionsOutsiderange - active collections dated not between the * start and end timestamps and included in the CMR mock - * + extraESCollections - collections within the timestamp range, but excluded - * from CMR mock. (only in ES) - * + extraESCollectionsOutOfRange - collections outside the timestamp range and - * excluded from CMR mock. (only in ES out of range) - * + extraCmrCollections - collections not in ES but returned by the CMR mock. + * + extraPgCollections - collections within the timestamp range, but excluded + * from CMR mock + * + extraPgCollectionsOutOfRange - collections outside the timestamp range and + * excluded from CMR mock + * + extraCmrCollections - collections not in pg but returned by the CMR mock */ -const setupElasticAndCMRForTests = async ({ t, params = {} }) => { +const setupDatabaseAndCMRForTests = async ({ t, params = {} }) => { const dataBuckets = range(2).map(() => randomId('bucket')); await Promise.all( dataBuckets.map((bucket) => @@ -261,8 +289,8 @@ const setupElasticAndCMRForTests = async ({ t, params = {} }) => { const { numMatchingCollections = randomBetween(10, 15), numMatchingCollectionsOutOfRange = randomBetween(5, 10), - numExtraESCollections = randomBetween(5, 10), - numExtraESCollectionsOutOfRange = randomBetween(5, 10), + numExtraPgCollections = randomBetween(5, 10), + numExtraPgCollectionsOutOfRange = randomBetween(5, 10), numExtraCmrCollections = randomBetween(5, 10), } = params; @@ -271,32 +299,37 @@ const setupElasticAndCMRForTests = async ({ t, params = {} }) => { const endTimestamp = new Date('2020-07-01T00:00:00.000Z').getTime(); const monthLater = moment(endTimestamp).add(1, 'month').valueOf(); - // Create collections that are in sync ES/CMR during the time period + // Create collections that are in sync pg/CMR during the time period const matchingCollections = range(numMatchingCollections).map((r) => ({ + ...requiredStaticCollectionFields, name: randomId(`name${r}-`), version: randomId('vers'), updatedAt: randomTimeBetween(startTimestamp, endTimestamp), })); - // Create collections in sync ES/CMR outside of the timestamps range + // Create collections in sync pg/CMR outside of the timestamps range const matchingCollectionsOutsideRange = range(numMatchingCollectionsOutOfRange).map((r) => ({ + ...requiredStaticCollectionFields, name: randomId(`name${r}-`), version: randomId('vers'), updatedAt: randomTimeBetween(monthEarlier, startTimestamp - 1), })); - // Create collections in ES only within the timestamp range - const extraESCollections = range(numExtraESCollections).map((r) => ({ - name: randomId(`extraES${r}-`), + // Create collections in pg only within the timestamp range + const extraPgCollections = range(numExtraPgCollections).map((r) => ({ + ...requiredStaticCollectionFields, + name: randomId(`extraPg${r}-`), version: randomId('vers'), updatedAt: randomTimeBetween(startTimestamp, endTimestamp), })); - // Create collections in ES only outside of the timestamp range - const extraESCollectionsOutOfRange = range(numExtraESCollectionsOutOfRange).map((r) => ({ - name: randomId(`extraES${r}-`), + // Create collections in pg only outside of the timestamp range + const extraPgCollectionsOutOfRange = range(numExtraPgCollectionsOutOfRange).map((r) => ({ + ...requiredStaticCollectionFields, + name: randomId(`extraPg${r}-`), version: randomId('vers'), updatedAt: randomTimeBetween(endTimestamp + 1, monthLater), })); // create extra cmr collections that fall inside of the range. const extraCmrCollections = range(numExtraCmrCollections).map((r) => ({ + ...requiredStaticCollectionFields, name: randomId(`extraCmr${r}-`), version: randomId('vers'), updatedAt: randomTimeBetween(startTimestamp, endTimestamp), @@ -318,48 +351,59 @@ const setupElasticAndCMRForTests = async ({ t, params = {} }) => { cmrSearchStub.withArgs('collections').onCall(1).resolves([]); cmrSearchStub.withArgs('granules').resolves([]); - await storeCollectionsToElasticsearch( - matchingCollections - .concat(matchingCollectionsOutsideRange) - .concat(extraESCollections) - .concat(extraESCollectionsOutOfRange) - ); + const { collections: createdCollections, granules: collectionGranules } = + await storeCollectionsWithGranuleToPostgres( + matchingCollections + .concat(matchingCollectionsOutsideRange) + .concat(extraPgCollections) + .concat(extraPgCollectionsOutOfRange), + t.context + ); + const mappedProviders = {}; + createdCollections.forEach((collection) => { + mappedProviders[ + constructCollectionId(collection.name, collection.version) + ] = collection.providerName; + }); return { startTimestamp, endTimestamp, matchingCollections, matchingCollectionsOutsideRange, - extraESCollections, - extraESCollectionsOutOfRange, + extraPgCollections, + extraPgCollectionsOutOfRange, extraCmrCollections, + collectionGranules, + mappedProviders, }; }; -test.before(async (t) => { - process.env = { - ...process.env, - ...localStackConnectionEnv, - PG_DATABASE: testDbName, - }; +test.before(async () => { process.env.cmr_password_secret_name = randomId('cmr-secret-name'); + process.env.DISTRIBUTION_ENDPOINT = 'TEST_ENDPOINT'; await awsServices.secretsManager().createSecret({ Name: process.env.cmr_password_secret_name, SecretString: randomId('cmr-password'), }); - const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); +}); + +test.beforeEach(async (t) => { + t.context.testDbName = `create_rec_reports_${cryptoRandomString({ length: 10 })}`; + process.env = { + ...process.env, + ...localStackConnectionEnv, + PG_DATABASE: t.context.testDbName, + }; + const { knex, knexAdmin } = await generateLocalTestDb(t.context.testDbName, migrationDir); t.context.knex = knex; t.context.knexAdmin = knexAdmin; - + t.context.providerPgModel = new ProviderPgModel(); t.context.collectionPgModel = new CollectionPgModel(); t.context.executionPgModel = new ExecutionPgModel(); t.context.filePgModel = new FilePgModel(); t.context.granulePgModel = new GranulePgModel(); -}); - -test.beforeEach(async (t) => { - process.env.ReconciliationReportsTable = randomId('reconciliationTable'); - + t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); t.context.bucketsToCleanup = []; t.context.stackName = randomId('stack'); t.context.systemBucket = randomId('bucket'); @@ -368,26 +412,19 @@ test.beforeEach(async (t) => { await awsServices.s3().createBucket({ Bucket: t.context.systemBucket }) .then(() => t.context.bucketsToCleanup.push(t.context.systemBucket)); - await new models.ReconciliationReport().createTable(); - const cmrSearchStub = sinon.stub(CMR.prototype, 'searchConcept'); cmrSearchStub.withArgs('collections').resolves([]); cmrSearchStub.withArgs('granules').resolves([]); - esAlias = randomId('esalias'); - esIndex = randomId('esindex'); - process.env.ES_INDEX = esAlias; - await bootstrapElasticSearch({ - host: 'fakehost', - index: esIndex, - alias: esAlias, - }); - esClient = await getEsClient(); - t.context.esReportClient = new Search( - {}, - 'reconciliationReport', - process.env.ES_INDEX - ); + // write 4 providers to the database + t.context.providers = await Promise.all(new Array(4).fill().map(async () => { + const [pgProvider] = await t.context.providerPgModel.create( + t.context.knex, + fakeProviderRecordFactory(), + ['cumulus_id', 'name'] + ); + return pgProvider; + })); t.context.execution = fakeExecutionRecordFactory(); const [pgExecution] = await t.context.executionPgModel.create( @@ -400,30 +437,26 @@ test.beforeEach(async (t) => { test.afterEach.always(async (t) => { await Promise.all( - flatten([ - t.context.bucketsToCleanup.map(recursivelyDeleteS3Bucket), - new models.ReconciliationReport().deleteTable(), - ]) + flatten(t.context.bucketsToCleanup.map(recursivelyDeleteS3Bucket)) ); await t.context.executionPgModel.delete( t.context.knex, { cumulus_id: t.context.executionCumulusId } ); CMR.prototype.searchConcept.restore(); - await esClient.client.indices.delete({ index: esIndex }); + await destroyLocalTestDb({ + knex: t.context.knex, + knexAdmin: t.context.knexAdmin, + testDbName: t.context.testDbName, + }); }); -test.after.always(async (t) => { +test.after.always(async () => { await awsServices.secretsManager().deleteSecret({ SecretId: process.env.cmr_password_secret_name, ForceDeleteWithoutRecovery: true, }); delete process.env.cmr_password_secret_name; - await destroyLocalTestDb({ - knex: t.context.knex, - knexAdmin: t.context.knexAdmin, - testDbName, - }); }); test.serial('Generates valid reconciliation report for no buckets', async (t) => { @@ -462,73 +495,10 @@ test.serial('Generates valid reconciliation report for no buckets', async (t) => t.true(createStartTime <= createEndTime); t.is(report.reportStartTime, (new Date(startTimestamp)).toISOString()); t.is(report.reportEndTime, (new Date(endTimestamp)).toISOString()); - - const esRecord = await t.context.esReportClient.get(reportRecord.name); - t.like(esRecord, reportRecord); }); test.serial('Generates valid GNF reconciliation report when everything is in sync', async (t) => { - const { filePgModel, granulePgModel, knex } = t.context; - - const dataBuckets = range(2).map(() => randomId('bucket')); - await Promise.all(dataBuckets.map((bucket) => - createBucket(bucket) - .then(() => t.context.bucketsToCleanup.push(bucket)))); - - // Write the buckets config to S3 - await storeBucketsConfigToS3( - dataBuckets, - t.context.systemBucket, - t.context.stackName - ); - - // Create collections that are in sync - const matchingColls = range(10).map(() => ({ - name: randomId('name'), - version: randomId('vers'), - })); - await storeCollectionsToElasticsearch(matchingColls); - - const collection = fakeCollectionRecordFactory({ - name: matchingColls[0].name, - version: matchingColls[0].version, - }); - const [pgCollection] = await t.context.collectionPgModel.create( - t.context.knex, - collection - ); - const collectionCumulusId = pgCollection.cumulus_id; - - // Create random files - const pgGranules = await granulePgModel.insert( - knex, - range(10).map(() => fakeGranuleRecordFactory({ - collection_cumulus_id: collectionCumulusId, - })) - ); - const files = range(10).map((i) => ({ - bucket: dataBuckets[i % dataBuckets.length], - key: randomId('key'), - granule_cumulus_id: pgGranules[i].cumulus_id, - })); - - // Store the files to S3 and DynamoDB - await Promise.all([ - storeFilesToS3(files), - filePgModel.insert(knex, files), - ]); - - const cmrCollections = sortBy(matchingColls, ['name', 'version']) - .map((cmrCollection) => ({ - umm: { ShortName: cmrCollection.name, Version: cmrCollection.version }, - })); - - CMR.prototype.searchConcept.restore(); - const cmrSearchStub = sinon.stub(CMR.prototype, 'searchConcept'); - cmrSearchStub.withArgs('collections').onCall(0).resolves(cmrCollections); - cmrSearchStub.withArgs('collections').onCall(1).resolves([]); - cmrSearchStub.withArgs('granules').resolves([]); - + const { files, matchingColls } = await generateRandomGranules(t); const event = { systemBucket: t.context.systemBucket, stackName: t.context.stackName, @@ -561,74 +531,10 @@ test.serial('Generates valid GNF reconciliation report when everything is in syn const createStartTime = moment(report.createStartTime); const createEndTime = moment(report.createEndTime); t.true(createStartTime <= createEndTime); - - const esRecord = await t.context.esReportClient.get(reportRecord.name); - t.like(esRecord, reportRecord); }); test.serial('Generates a valid Inventory reconciliation report when everything is in sync', async (t) => { - const { filePgModel, granulePgModel, knex } = t.context; - - const dataBuckets = range(2).map(() => randomId('bucket')); - await Promise.all(dataBuckets.map((bucket) => - createBucket(bucket) - .then(() => t.context.bucketsToCleanup.push(bucket)))); - - // Write the buckets config to S3 - await storeBucketsConfigToS3( - dataBuckets, - t.context.systemBucket, - t.context.stackName - ); - - // Create collections that are in sync - const matchingColls = range(10).map(() => ({ - name: randomId('name'), - version: randomId('vers'), - })); - await storeCollectionsToElasticsearch(matchingColls); - - const collection = fakeCollectionRecordFactory({ - name: matchingColls[0].name, - version: matchingColls[0].version, - }); - const [pgCollection] = await t.context.collectionPgModel.create( - t.context.knex, - collection - ); - const collectionCumulusId = pgCollection.cumulus_id; - - // Create random files - const pgGranules = await granulePgModel.insert( - knex, - range(10).map(() => fakeGranuleRecordFactory({ - collection_cumulus_id: collectionCumulusId, - })) - ); - const files = range(10).map((i) => ({ - bucket: dataBuckets[i % dataBuckets.length], - key: randomId('key'), - granule_cumulus_id: pgGranules[i].cumulus_id, - })); - - // Store the files to S3 and DynamoDB - await Promise.all([ - storeFilesToS3(files), - filePgModel.insert(knex, files), - ]); - - const cmrCollections = sortBy(matchingColls, ['name', 'version']) - .map((cmrCollection) => ({ - umm: { ShortName: cmrCollection.name, Version: cmrCollection.version }, - })); - - CMR.prototype.searchConcept.restore(); - const cmrSearchStub = sinon.stub(CMR.prototype, 'searchConcept'); - cmrSearchStub.withArgs('collections').onCall(0).resolves(cmrCollections); - cmrSearchStub.withArgs('collections').onCall(1).resolves([]); - cmrSearchStub.withArgs('granules').resolves([]); - - await storeCollectionsToElasticsearch(matchingColls); + const { files, matchingColls } = await generateRandomGranules(t); const event = { systemBucket: t.context.systemBucket, @@ -644,7 +550,7 @@ test.serial('Generates a valid Inventory reconciliation report when everything i const collectionsInCumulusCmr = report.collectionsInCumulusCmr; t.is(report.status, 'SUCCESS'); - t.is(filesInCumulus.okCountByGranule, undefined); + t.deepEqual(filesInCumulus.okCountByGranule, {}); t.is(report.error, undefined); t.is(filesInCumulus.okCount, files.length); @@ -660,46 +566,15 @@ test.serial('Generates a valid Inventory reconciliation report when everything i }); test.serial('Generates valid reconciliation report when there are extra internal S3 objects', async (t) => { - const { filePgModel, granulePgModel, knex } = t.context; - - const collection = fakeCollectionRecordFactory(); - const [pgCollection] = await t.context.collectionPgModel.create( - t.context.knex, - collection - ); - const collectionCumulusId = pgCollection.cumulus_id; - - const dataBuckets = range(2).map(() => randomId('bucket')); - await Promise.all(dataBuckets.map((bucket) => - createBucket(bucket) - .then(() => t.context.bucketsToCleanup.push(bucket)))); - - // Write the buckets config to S3 - await storeBucketsConfigToS3( - dataBuckets, - t.context.systemBucket, - t.context.stackName - ); - - // Create files that are in sync - const pgGranules = await granulePgModel.insert( - knex, - range(10).map(() => fakeGranuleRecordFactory({ - collection_cumulus_id: collectionCumulusId, - })) - ); - const matchingFiles = range(10).map((i) => ({ - bucket: sample(dataBuckets), - key: randomId('key'), - granule_cumulus_id: pgGranules[i].cumulus_id, - })); + const { dataBuckets, files } = await generateRandomGranules(t, { + collectionRange: 1, + stubCmr: false, + }); const extraS3File1 = { bucket: sample(dataBuckets), key: randomId('key') }; const extraS3File2 = { bucket: sample(dataBuckets), key: randomId('key') }; - // Store the files to S3 and Elasticsearch - await storeFilesToS3(matchingFiles.concat([extraS3File1, extraS3File2])); - await filePgModel.insert(knex, matchingFiles); + await storeFilesToS3(files.concat([extraS3File1, extraS3File2])); const event = { systemBucket: t.context.systemBucket, @@ -714,7 +589,7 @@ test.serial('Generates valid reconciliation report when there are extra internal const filesInCumulus = report.filesInCumulus; t.is(report.status, 'SUCCESS'); t.is(report.error, undefined); - t.is(filesInCumulus.okCount, matchingFiles.length); + t.is(filesInCumulus.okCount, files.length); const granuleIds = Object.keys(filesInCumulus.okCountByGranule); granuleIds.forEach((granuleId) => { @@ -733,61 +608,25 @@ test.serial('Generates valid reconciliation report when there are extra internal t.true(createStartTime <= createEndTime); }); -test.serial('Generates valid reconciliation report when there are extra internal DynamoDB objects', async (t) => { - const { filePgModel, granulePgModel, knex } = t.context; - - const dataBuckets = range(2).map(() => randomString()); - await Promise.all(dataBuckets.map((bucket) => - createBucket(bucket) - .then(() => t.context.bucketsToCleanup.push(bucket)))); - - // Write the buckets config to S3 - await storeBucketsConfigToS3( - dataBuckets, - t.context.systemBucket, - t.context.stackName - ); - - const collection = fakeCollectionRecordFactory(); - const [pgCollection] = await t.context.collectionPgModel.create( - t.context.knex, - collection - ); - const collectionCumulusId = pgCollection.cumulus_id; - - // Create files that are in sync - const granules = range(12).map(() => fakeGranuleRecordFactory({ - collection_cumulus_id: collectionCumulusId, - })); - const pgGranules = await granulePgModel.insert( - knex, - granules - ); - const matchingFiles = range(10).map((i) => ({ - bucket: sample(dataBuckets), - key: randomId('key'), - granule_cumulus_id: pgGranules[i].cumulus_id, - })); +test.serial('Generates valid reconciliation report when there are extra internal Postgres objects', async (t) => { + const { granules, files, dataBuckets } = await generateRandomGranules(t, { + collectionRange: 1, + granuleRange: 12, + }); + const [extraFileGranule1, extraFileGranule2] = granules.slice(10, 12); const extraDbFile1 = { bucket: sample(dataBuckets), key: randomString(), - granule_cumulus_id: pgGranules[10].cumulus_id, - granule_id: granules[10].granule_id, + granule_cumulus_id: extraFileGranule1.cumulus_id, }; const extraDbFile2 = { bucket: sample(dataBuckets), key: randomString(), - granule_cumulus_id: pgGranules[11].cumulus_id, - granule_id: granules[11].granule_id, + granule_cumulus_id: extraFileGranule2.cumulus_id, }; - // Store the files to S3 and DynamoDB - await storeFilesToS3(matchingFiles); - await filePgModel.insert(knex, matchingFiles.concat([ - omit(extraDbFile1, 'granule_id'), - omit(extraDbFile2, 'granule_id'), - ])); + await t.context.filePgModel.insert(t.context.knex, [extraDbFile1, extraDbFile2]); const event = { systemBucket: t.context.systemBucket, @@ -802,7 +641,7 @@ test.serial('Generates valid reconciliation report when there are extra internal const filesInCumulus = report.filesInCumulus; t.is(report.status, 'SUCCESS'); t.is(report.error, undefined); - t.is(filesInCumulus.okCount, matchingFiles.length); + t.is(filesInCumulus.okCount, files.length); t.is(filesInCumulus.onlyInS3.length, 0); const totalOkCount = Object.values(filesInCumulus.okCountByGranule).reduce( @@ -813,17 +652,17 @@ test.serial('Generates valid reconciliation report when there are extra internal t.is(filesInCumulus.onlyInDb.length, 2); t.truthy(filesInCumulus.onlyInDb.find((f) => f.uri === buildS3Uri(extraDbFile1.bucket, extraDbFile1.key) - && f.granuleId === extraDbFile1.granule_id)); + && f.granuleId === extraFileGranule1.granule_id)); t.truthy(filesInCumulus.onlyInDb.find((f) => f.uri === buildS3Uri(extraDbFile2.bucket, extraDbFile2.key) - && f.granuleId === extraDbFile2.granule_id)); + && f.granuleId === extraFileGranule2.granule_id)); const createStartTime = moment(report.createStartTime); const createEndTime = moment(report.createEndTime); t.true(createStartTime <= createEndTime); }); -test.serial('Generates valid reconciliation report when internally, there are both extra DynamoDB and extra S3 files', async (t) => { +test.serial('Generates valid reconciliation report when internally, there are both extra postgres and extra S3 files', async (t) => { const { filePgModel, granulePgModel, knex } = t.context; const collection = fakeCollectionRecordFactory(); @@ -875,7 +714,7 @@ test.serial('Generates valid reconciliation report when internally, there are bo granule_id: granules[11].granule_id, }; - // Store the files to S3 and DynamoDB + // Store the files to S3 and postgres await storeFilesToS3(matchingFiles.concat([extraS3File1, extraS3File2])); await filePgModel.insert(knex, matchingFiles.concat([ omit(extraDbFile1, 'granule_id'), @@ -919,13 +758,13 @@ test.serial('Generates valid reconciliation report when internally, there are bo t.true(createStartTime <= createEndTime); }); -test.serial('Generates valid reconciliation report when there are both extra ES and CMR collections', async (t) => { +test.serial('Generates valid reconciliation report when there are both extra postGres and CMR collections', async (t) => { const params = { numMatchingCollectionsOutOfRange: 0, - numExtraESCollectionsOutOfRange: 0, + numExtraPgCollectionsOutOfRange: 0, }; - const setupVars = await setupElasticAndCMRForTests({ t, params }); + const setupVars = await setupDatabaseAndCMRForTests({ t, params }); const event = { systemBucket: t.context.systemBucket, @@ -942,8 +781,8 @@ test.serial('Generates valid reconciliation report when there are both extra ES t.is(report.error, undefined); t.is(collectionsInCumulusCmr.okCount, setupVars.matchingCollections.length); - t.is(collectionsInCumulusCmr.onlyInCumulus.length, setupVars.extraESCollections.length); - setupVars.extraESCollections.map((collection) => + t.is(collectionsInCumulusCmr.onlyInCumulus.length, setupVars.extraPgCollections.length); + setupVars.extraPgCollections.map((collection) => t.true(collectionsInCumulusCmr.onlyInCumulus .includes(constructCollectionId(collection.name, collection.version)))); @@ -958,9 +797,9 @@ test.serial('Generates valid reconciliation report when there are both extra ES }); test.serial( - 'With input time params, generates a valid filtered reconciliation report, when there are extra cumulus/ES and CMR collections', + 'With input time params, generates a valid filtered reconciliation report, when there are extra cumulus database and CMR collections', async (t) => { - const { startTimestamp, endTimestamp, ...setupVars } = await setupElasticAndCMRForTests({ t }); + const { startTimestamp, endTimestamp, ...setupVars } = await setupDatabaseAndCMRForTests({ t }); const event = { systemBucket: t.context.systemBucket, @@ -978,14 +817,14 @@ test.serial( t.is(report.error, undefined); t.is(collectionsInCumulusCmr.okCount, setupVars.matchingCollections.length); - t.is(collectionsInCumulusCmr.onlyInCumulus.length, setupVars.extraESCollections.length); + t.is(collectionsInCumulusCmr.onlyInCumulus.length, setupVars.extraPgCollections.length); // Each extra collection in timerange is included - setupVars.extraESCollections.map((collection) => + setupVars.extraPgCollections.map((collection) => t.true(collectionsInCumulusCmr.onlyInCumulus .includes(constructCollectionId(collection.name, collection.version)))); // No collections that were out of timestamp are included - setupVars.extraESCollectionsOutOfRange.map((collection) => + setupVars.extraPgCollectionsOutOfRange.map((collection) => t.false(collectionsInCumulusCmr.onlyInCumulus .includes(constructCollectionId(collection.name, collection.version)))); @@ -1006,7 +845,7 @@ test.serial( ); test.serial( - 'With location param as S3, generates a valid reconciliation report for only S3 and DynamoDB', + 'With location param as S3, generates a valid reconciliation report for only S3 and postgres', async (t) => { const { filePgModel, granulePgModel, knex } = t.context; @@ -1084,10 +923,10 @@ test.serial( async (t) => { const params = { numMatchingCollectionsOutOfRange: 0, - numExtraESCollectionsOutOfRange: 0, + numExtraPgCollectionsOutOfRange: 0, }; - const setupVars = await setupElasticAndCMRForTests({ t, params }); + const setupVars = await setupDatabaseAndCMRForTests({ t, params }); const event = { systemBucket: t.context.systemBucket, @@ -1105,8 +944,8 @@ test.serial( t.is(collectionsInCumulusCmr.okCount, setupVars.matchingCollections.length); t.is(report.filesInCumulus.okCount, 0); - t.is(collectionsInCumulusCmr.onlyInCumulus.length, setupVars.extraESCollections.length); - setupVars.extraESCollections.map((collection) => + t.is(collectionsInCumulusCmr.onlyInCumulus.length, setupVars.extraPgCollections.length); + setupVars.extraPgCollections.map((collection) => t.true(collectionsInCumulusCmr.onlyInCumulus .includes(constructCollectionId(collection.name, collection.version)))); @@ -1118,9 +957,9 @@ test.serial( ); test.serial( - 'Generates valid reconciliation report without time params and there are extra cumulus/ES and CMR collections', + 'Generates valid reconciliation report without time params and there are extra cumulus DB and CMR collections', async (t) => { - const setupVars = await setupElasticAndCMRForTests({ t }); + const setupVars = await setupDatabaseAndCMRForTests({ t }); const eventNoTimeStamps = { systemBucket: t.context.systemBucket, @@ -1141,15 +980,15 @@ test.serial( setupVars.matchingCollections.length + setupVars.matchingCollectionsOutsideRange.length ); - // all extra ES collections are found + // all extra DB collections are found t.is( collectionsInCumulusCmr.onlyInCumulus.length, - setupVars.extraESCollections.length + setupVars.extraESCollectionsOutOfRange.length + setupVars.extraPgCollections.length + setupVars.extraPgCollectionsOutOfRange.length ); - setupVars.extraESCollections.map((collection) => + setupVars.extraPgCollections.map((collection) => t.true(collectionsInCumulusCmr.onlyInCumulus .includes(constructCollectionId(collection.name, collection.version)))); - setupVars.extraESCollectionsOutOfRange.map((collection) => + setupVars.extraPgCollectionsOutOfRange.map((collection) => t.true(collectionsInCumulusCmr.onlyInCumulus .includes(constructCollectionId(collection.name, collection.version)))); @@ -1165,15 +1004,15 @@ test.serial( ); test.serial( - 'Generates valid ONE WAY reconciliation report with time params and filters by collectionIds when there are extra cumulus/ES and CMR collections', + 'Generates valid ONE WAY reconciliation report with time params and filters by collectionIds when there are extra cumulus DB and CMR collections', async (t) => { - const { startTimestamp, endTimestamp, ...setupVars } = await setupElasticAndCMRForTests({ t }); + const { startTimestamp, endTimestamp, ...setupVars } = await setupDatabaseAndCMRForTests({ t }); const testCollection = [ setupVars.matchingCollections[3], setupVars.extraCmrCollections[1], - setupVars.extraESCollections[1], - setupVars.extraESCollectionsOutOfRange[0], + setupVars.extraPgCollections[1], + setupVars.extraPgCollectionsOutOfRange[0], ]; const collectionId = testCollection.map((c) => constructCollectionId(c.name, c.version)); @@ -1220,7 +1059,7 @@ test.serial( test.serial( 'When a collectionId is in both CMR and Cumulus a valid bi-directional reconciliation report is created.', async (t) => { - const setupVars = await setupElasticAndCMRForTests({ t }); + const setupVars = await setupDatabaseAndCMRForTests({ t }); const testCollection = setupVars.matchingCollections[3]; console.log(`testCollection: ${JSON.stringify(testCollection)}`); @@ -1250,12 +1089,12 @@ test.serial( test.serial( 'When an array of collectionId exists only in CMR, creates a valid bi-directional reconciliation report.', async (t) => { - const setupVars = await setupElasticAndCMRForTests({ t }); + const setupVars = await setupDatabaseAndCMRForTests({ t }); const testCollection = [ setupVars.extraCmrCollections[3], setupVars.matchingCollections[2], - setupVars.extraESCollections[1], + setupVars.extraPgCollections[1], ]; const collectionId = testCollection.map((c) => constructCollectionId(c.name, c.version)); console.log(`testCollection: ${JSON.stringify(collectionId)}`); @@ -1288,9 +1127,9 @@ test.serial( test.serial( 'When a filtered collectionId exists only in Cumulus, generates a valid bi-directional reconciliation report.', async (t) => { - const setupVars = await setupElasticAndCMRForTests({ t }); + const setupVars = await setupDatabaseAndCMRForTests({ t }); - const testCollection = setupVars.extraESCollections[3]; + const testCollection = setupVars.extraPgCollections[3]; console.log(`testCollection: ${JSON.stringify(testCollection)}`); const event = { @@ -1322,19 +1161,23 @@ test.serial( ); test.serial( - 'Generates valid ONE WAY reconciliation report with time params and filters by granuleIds when there are extra cumulus/ES and CMR collections', + 'Generates valid ONE WAY reconciliation report with time params and filters by granuleIds when there are extra cumulus/pg and CMR collections', async (t) => { - const { startTimestamp, endTimestamp, ...setupVars } = await setupElasticAndCMRForTests({ t }); + const { startTimestamp, endTimestamp, ...setupVars } = await setupDatabaseAndCMRForTests({ t }); const testCollection = [ setupVars.matchingCollections[3], setupVars.extraCmrCollections[1], - setupVars.extraESCollections[1], - setupVars.extraESCollectionsOutOfRange[0], + setupVars.extraPgCollections[1], + setupVars.extraPgCollectionsOutOfRange[0], ]; const testCollectionIds = testCollection.map((c) => constructCollectionId(c.name, c.version)); - const testGranuleIds = await granuleIdsFromCollectionIds(testCollectionIds); + + //set testGranuleIds to be all setupVars.collectionGranules that are in testCollectionIds + const testGranuleIds = setupVars.collectionGranules + .filter((g) => testCollectionIds.includes(g.collectionId)) + .map((g) => g.granule_id); console.log(`granuleIds: ${JSON.stringify(testGranuleIds)}`); @@ -1353,14 +1196,12 @@ test.serial( const collectionsInCumulusCmr = report.collectionsInCumulusCmr; t.is(report.status, 'SUCCESS'); t.is(report.error, undefined); - t.is(collectionsInCumulusCmr.okCount, 1); // cumulus filters collections by granuleId and only returned test one t.is(collectionsInCumulusCmr.onlyInCumulus.length, 1); t.true(collectionsInCumulusCmr.onlyInCumulus.includes(testCollectionIds[2])); - // ONE WAY only comparison because of input timestampes t.is(collectionsInCumulusCmr.onlyInCmr.length, 0); const reportStartTime = report.reportStartTime; @@ -1379,16 +1220,18 @@ test.serial( test.serial( 'When an array of granuleId exists, creates a valid one-way reconciliation report.', async (t) => { - const setupVars = await setupElasticAndCMRForTests({ t }); + const setupVars = await setupDatabaseAndCMRForTests({ t }); const testCollection = [ setupVars.extraCmrCollections[3], setupVars.matchingCollections[2], - setupVars.extraESCollections[1], + setupVars.extraPgCollections[1], ]; const testCollectionIds = testCollection.map((c) => constructCollectionId(c.name, c.version)); - const testGranuleIds = await granuleIdsFromCollectionIds(testCollectionIds); + const testGranuleIds = setupVars.collectionGranules + .filter((g) => testCollectionIds.includes(g.collectionId)) + .map((g) => g.granule_id); console.log(`testGranuleIds: ${JSON.stringify(testGranuleIds)}`); @@ -1402,11 +1245,11 @@ test.serial( t.is(reportRecord.status, 'Generated'); const report = await fetchCompletedReport(reportRecord); - const collectionsInCumulusCmr = report.collectionsInCumulusCmr; t.is(report.status, 'SUCCESS'); t.is(report.error, undefined); // Filtered by input granuleIds + const collectionsInCumulusCmr = report.collectionsInCumulusCmr; t.is(collectionsInCumulusCmr.okCount, 1); t.is(collectionsInCumulusCmr.onlyInCumulus.length, 1); t.true(collectionsInCumulusCmr.onlyInCumulus.includes(testCollectionIds[2])); @@ -1421,16 +1264,19 @@ test.serial( test.serial( 'When an array of providers exists, creates a valid one-way reconciliation report.', async (t) => { - const setupVars = await setupElasticAndCMRForTests({ t }); + const setupVars = await setupDatabaseAndCMRForTests({ t }); + // TODO: collections work! Failures should be granules now. const testCollection = [ setupVars.extraCmrCollections[3], setupVars.matchingCollections[2], - setupVars.extraESCollections[1], + setupVars.extraPgCollections[1], ]; const testCollectionIds = testCollection.map((c) => constructCollectionId(c.name, c.version)); - const testProviders = await providersFromCollectionIds(testCollectionIds); + const testProviders = compact(testCollection.map( + (c) => setupVars.mappedProviders[constructCollectionId(c.name, c.version)] + )); const event = { systemBucket: t.context.systemBucket, @@ -1452,7 +1298,6 @@ test.serial( t.is(collectionsInCumulusCmr.okCount, 1); t.is(collectionsInCumulusCmr.onlyInCumulus.length, 1); t.true(collectionsInCumulusCmr.onlyInCumulus.includes(testCollectionIds[2])); - t.is(granulesInCumulusCmr.okCount, 0); t.is(granulesInCumulusCmr.onlyInCumulus.length, 1); @@ -1465,11 +1310,23 @@ test.serial( } ); -test.serial('reconciliationReportForGranules reports discrepancy of granule holdings in CUMULUS and CMR', async (t) => { +// TODO - this test feels *wholly* not great are we relying on spec tests? +// TODO - add test for *multiple* collections, etc. // SPEC TESTS? +test.serial('reconciliationReportForGranules reports discrepancy of granule holdings in CUMULUS and CMR for a single collection', async (t) => { + // TODO - common methods? const shortName = randomString(); const version = randomString(); const collectionId = constructCollectionId(shortName, version); + const postgresCollectionRecord = fakeCollectionRecordFactory({ + name: shortName, + version, + }); + await t.context.collectionPgModel.create( + t.context.knex, + postgresCollectionRecord + ); + // create granules that are in sync const matchingGrans = range(10).map(() => fakeGranuleFactoryV2({ collectionId: collectionId, status: 'completed', files: [] })); @@ -1495,13 +1352,23 @@ test.serial('reconciliationReportForGranules reports discrepancy of granule hold cmrSearchStub.withArgs('granules').onCall(0).resolves(cmrGranules); cmrSearchStub.withArgs('granules').onCall(1).resolves([]); - await storeGranulesToElasticsearch(matchingGrans.concat(extraDbGrans)); + await Promise.all( + matchingGrans + .concat(extraDbGrans) + .map(async (granule) => { + const pgGranule = await translateApiGranuleToPostgresGranule({ + dynamoRecord: granule, + knexOrTransaction: t.context.knex, + }); + return await t.context.granulePgModel.create(t.context.knex, pgGranule); + }) + ); const { granulesReport, filesReport } = await reconciliationReportForGranules({ collectionId, bucketsConfig: new BucketsConfig({}), distributionBucketMap: {}, - recReportParams: {}, + recReportParams: { knex: t.context.knex }, }); t.is(granulesReport.okCount, 10); @@ -1867,90 +1734,20 @@ test.serial('When report creation fails, reconciliation report status is set to }; await t.throwsAsync(handler(event)); - const reportRecord = await new models.ReconciliationReport().get({ name: reportName }); - t.is(reportRecord.status, 'Failed'); - t.is(reportRecord.type, 'Inventory'); + + const reportPgRecord = await t.context.reconciliationReportPgModel.get( + t.context.knex, { name: reportName } + ); + // reconciliation report lambda outputs the translated API version, not the PG version, so + // it should be translated for comparison + const reportApiRecord = translatePostgresReconReportToApiReconReport(reportPgRecord); + t.is(reportApiRecord.status, 'Failed'); + t.is(reportApiRecord.type, 'Inventory'); const reportKey = `${t.context.stackName}/reconciliation-reports/${reportName}.json`; const report = await getJsonS3Object(t.context.systemBucket, reportKey); t.is(report.status, 'Failed'); t.truthy(report.error); - - const esRecord = await t.context.esReportClient.get(reportRecord.name); - t.like(esRecord, reportRecord); -}); - -test.serial('A valid internal reconciliation report is generated when ES and DB are in sync', async (t) => { - const { - knex, - execution, - executionCumulusId, - } = t.context; - - const collection = fakeCollectionRecordFactory(); - const collectionId = constructCollectionId( - collection.name, - collection.version - ); - const [pgCollection] = await t.context.collectionPgModel.create( - t.context.knex, - collection - ); - await indexer.indexCollection( - esClient, - translatePostgresCollectionToApiCollection(pgCollection), - esAlias - ); - - const matchingGrans = range(10).map(() => fakeGranuleFactoryV2({ - collectionId, - execution: execution.url, - })); - await Promise.all( - matchingGrans.map(async (gran) => { - await indexer.indexGranule(esClient, gran, esAlias); - const pgGranule = await translateApiGranuleToPostgresGranule({ - dynamoRecord: gran, - knexOrTransaction: knex, - }); - await upsertGranuleWithExecutionJoinRecord({ - executionCumulusId, - granule: pgGranule, - knexTransaction: knex, - }); - }) - ); - - const event = { - systemBucket: t.context.systemBucket, - stackName: t.context.stackName, - reportType: 'Internal', - reportName: randomId('reportName'), - collectionId, - startTimestamp: moment.utc().subtract(1, 'hour').format(), - endTimestamp: moment.utc().add(1, 'hour').format(), - }; - - const reportRecord = await handler(event); - t.is(reportRecord.status, 'Generated'); - t.is(reportRecord.name, event.reportName); - t.is(reportRecord.type, event.reportType); - - const report = await fetchCompletedReport(reportRecord); - t.is(report.status, 'SUCCESS'); - t.is(report.error, undefined); - t.is(report.reportType, 'Internal'); - t.is(report.collections.okCount, 1); - t.is(report.collections.onlyInEs.length, 0); - t.is(report.collections.onlyInDb.length, 0); - t.is(report.collections.withConflicts.length, 0); - t.is(report.granules.okCount, 10); - t.is(report.granules.onlyInEs.length, 0); - t.is(report.granules.onlyInDb.length, 0); - t.is(report.granules.withConflicts.length, 0); - - const esRecord = await t.context.esReportClient.get(reportRecord.name); - t.like(esRecord, reportRecord); }); test.serial('Creates a valid Granule Inventory report', async (t) => { @@ -1969,12 +1766,6 @@ test.serial('Creates a valid Granule Inventory report', async (t) => { collection ); const collectionCumulusId = pgCollection.cumulus_id; - await indexer.indexCollection( - esClient, - translatePostgresCollectionToApiCollection(pgCollection), - esAlias - ); - const matchingGrans = range(10).map(() => fakeGranuleRecordFactory({ collection_cumulus_id: collectionCumulusId, })); @@ -2004,18 +1795,16 @@ test.serial('Creates a valid Granule Inventory report', async (t) => { const header = '"granuleUr","collectionId","createdAt","startDateTime","endDateTime","status","updatedAt","published","provider"'; t.is(reportHeader, header); t.is(reportRows.length, 10); - - const esRecord = await t.context.esReportClient.get(reportRecord.name); - t.like(esRecord, reportRecord); }); test.serial('A valid ORCA Backup reconciliation report is generated', async (t) => { - const collection = fakeCollectionFactory({ + const { knex, collectionPgModel, granulePgModel, providerPgModel, filePgModel } = t.context; + const collection = fakeCollectionRecordFactory({ name: 'fakeCollection', version: 'v2', }); - await indexer.indexCollection(esClient, collection, esAlias); + await collectionPgModel.create(knex, collection); const collectionId = constructCollectionId(collection.name, collection.version); const matchingCumulusGran = { @@ -2047,7 +1836,23 @@ test.serial('A valid ORCA Backup reconciliation report is generated', async (t) ], }; - await indexer.indexGranule(esClient, matchingCumulusGran, esAlias); + await providerPgModel.create( + knex, + fakeProviderRecordFactory({ name: matchingCumulusGran.provider }) + ); + const pgGranule = await translateApiGranuleToPostgresGranule({ + dynamoRecord: matchingCumulusGran, + knexOrTransaction: knex, + }); + const pgGranuleRecord = await granulePgModel.create(knex, pgGranule); + await Promise.all( + matchingCumulusGran.files.map((file) => + filePgModel.create(knex, { + ...translateApiFiletoPostgresFile(file), + granule_cumulus_id: pgGranuleRecord[0].cumulus_id, + })) + ); + const searchOrcaStub = sinon.stub(ORCASearchCatalogQueue.prototype, 'searchOrca'); searchOrcaStub.resolves({ anotherPage: false, granules: [matchingOrcaGran] }); t.teardown(() => searchOrcaStub.restore()); @@ -2083,42 +1888,6 @@ test.serial('A valid ORCA Backup reconciliation report is generated', async (t) t.is(report.granules.onlyInCumulus.length, 0); t.is(report.granules.onlyInOrca.length, 0); t.is(report.granules.withConflicts.length, 0); - - const esRecord = await t.context.esReportClient.get(reportRecord.name); - t.like(esRecord, reportRecord); -}); - -test.serial('Internal Reconciliation report JSON is formatted', async (t) => { - const matchingColls = range(5).map(() => fakeCollectionFactory()); - const collectionId = constructCollectionId(matchingColls[0].name, matchingColls[0].version); - const matchingGrans = range(10).map(() => fakeGranuleFactoryV2({ collectionId })); - await Promise.all( - matchingColls.map((collection) => indexer.indexCollection(esClient, collection, esAlias)) - ); - await Promise.all( - matchingGrans.map((gran) => indexer.indexGranule(esClient, gran, esAlias)) - ); - - const event = { - systemBucket: t.context.systemBucket, - stackName: t.context.stackName, - reportType: 'Internal', - reportName: randomId('reportName'), - collectionId, - startTimestamp: moment.utc().subtract(1, 'hour').format(), - endTimestamp: moment.utc().add(1, 'hour').format(), - }; - - const reportRecord = await handler(event); - - const formattedReport = await fetchCompletedReportString(reportRecord); - - // Force report to unformatted (single line) - const unformattedReportString = JSON.stringify(JSON.parse(formattedReport), undefined, 0); - const unformattedReportObj = JSON.parse(unformattedReportString); - - t.true(!unformattedReportString.includes('\n')); // validate unformatted report is on a single line - t.is(formattedReport, JSON.stringify(unformattedReportObj, undefined, 2)); }); test.serial('Inventory reconciliation report JSON is formatted', async (t) => { @@ -2163,7 +1932,7 @@ test.serial('Inventory reconciliation report JSON is formatted', async (t) => { cmrSearchStub.withArgs('collections').onCall(1).resolves([]); cmrSearchStub.withArgs('granules').resolves([]); - await storeCollectionsToElasticsearch(matchingColls); + await storeCollectionsWithGranuleToPostgres(matchingColls, t.context); const eventFormatted = { systemBucket: t.context.systemBucket, @@ -2214,15 +1983,30 @@ test.serial('When there is an error for an ORCA backup report, it throws', async { message: 'ORCA error' } ); - const reportRecord = await new models.ReconciliationReport().get({ name: reportName }); - t.is(reportRecord.status, 'Failed'); - t.is(reportRecord.type, event.reportType); + const reportPgRecord = await t.context.reconciliationReportPgModel.get( + t.context.knex, { name: reportName } + ); + // reconciliation report lambda outputs the translated API version, not the PG version, so + // it should be translated for comparison + const reportApiRecord = translatePostgresReconReportToApiReconReport(reportPgRecord); + t.is(reportApiRecord.status, 'Failed'); + t.is(reportApiRecord.type, event.reportType); const reportKey = `${t.context.stackName}/reconciliation-reports/${reportName}.json`; const report = await getJsonS3Object(t.context.systemBucket, reportKey); t.is(report.status, 'Failed'); t.is(report.reportType, event.reportType); +}); + +test.serial('Internal reconciliation report type throws an error', async (t) => { + const event = { + systemBucket: t.context.systemBucket, + stackName: t.context.stackName, + reportType: 'Internal', + }; - const esRecord = await t.context.esReportClient.get(reportRecord.name); - t.like(esRecord, reportRecord); + await t.throwsAsync( + handler(event), + { message: 'Internal Reconciliation Reports are no longer valid' } + ); }); diff --git a/packages/api/tests/lambdas/test-granule-inventory-report.js b/packages/api/tests/lambdas/test-granule-inventory-report.js index 074bf81f33c..6830310c23b 100644 --- a/packages/api/tests/lambdas/test-granule-inventory-report.js +++ b/packages/api/tests/lambdas/test-granule-inventory-report.js @@ -87,7 +87,7 @@ test.serial('Writes a file containing all granules to S3.', async (t) => { const reportKey = `${t.context.stackName}/reconciliation-reports/${reportRecordName}.csv`; const systemBucket = t.context.systemBucket; const reportParams = { - ...normalizeEvent({ reportType: 'Granule Inventory' }), + ...normalizeEvent({ reportType: 'Granule Inventory', stackName: 'TestStack' }), reportKey, systemBucket, knex: t.context.knex, @@ -165,6 +165,7 @@ test.serial('Writes a file containing a filtered set of granules to S3.', async collectionId, status, granuleId: 'test', + stackName: 'testStack', }), reportKey, systemBucket, diff --git a/packages/api/tests/lambdas/test-index-from-database.js b/packages/api/tests/lambdas/test-index-from-database.js deleted file mode 100644 index 3aa6a6dae06..00000000000 --- a/packages/api/tests/lambdas/test-index-from-database.js +++ /dev/null @@ -1,537 +0,0 @@ -'use strict'; - -const cryptoRandomString = require('crypto-random-string'); -const sinon = require('sinon'); -const test = require('ava'); -const omit = require('lodash/omit'); - -const awsServices = require('@cumulus/aws-client/services'); -const { - promiseS3Upload, - recursivelyDeleteS3Bucket, -} = require('@cumulus/aws-client/S3'); -const { randomString } = require('@cumulus/common/test-utils'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const indexer = require('@cumulus/es-client/indexer'); -const { EsClient, Search } = require('@cumulus/es-client/search'); -const { - CollectionPgModel, - destroyLocalTestDb, - ExecutionPgModel, - fakeCollectionRecordFactory, - fakeExecutionRecordFactory, - fakeGranuleRecordFactory, - fakePdrRecordFactory, - fakeProviderRecordFactory, - fakeRuleRecordFactory, - generateLocalTestDb, - GranulePgModel, - migrationDir, - PdrPgModel, - ProviderPgModel, - RulePgModel, - translatePostgresCollectionToApiCollection, - translatePostgresExecutionToApiExecution, - translatePostgresGranuleToApiGranule, - translatePostgresPdrToApiPdr, - translatePostgresProviderToApiProvider, - translatePostgresRuleToApiRule, -} = require('@cumulus/db'); - -const { - fakeReconciliationReportFactory, -} = require('../../lib/testUtils'); - -const models = require('../../models'); -const indexFromDatabase = require('../../lambdas/index-from-database'); -const { - getWorkflowList, -} = require('../../lib/testUtils'); - -const workflowList = getWorkflowList(); -process.env.ReconciliationReportsTable = randomString(); -const reconciliationReportModel = new models.ReconciliationReport(); - -// create all the variables needed across this test -process.env.system_bucket = randomString(); -process.env.stackName = randomString(); - -const reconciliationReportsTable = process.env.ReconciliationReportsTable; - -function sortAndFilter(input, omitList, sortKey) { - return input.map((r) => omit(r, omitList)) - .sort((a, b) => (a[sortKey] > b[sortKey] ? 1 : -1)); -} - -async function addFakeDynamoData(numItems, factory, model, factoryParams = {}) { - const items = []; - - /* eslint-disable no-await-in-loop */ - for (let i = 0; i < numItems; i += 1) { - const item = factory(factoryParams); - items.push(item); - await model.create(item); - } - /* eslint-enable no-await-in-loop */ - - return items; -} - -async function addFakeData(knex, numItems, factory, model, factoryParams = {}) { - const items = []; - for (let i = 0; i < numItems; i += 1) { - const item = factory(factoryParams); - items.push(model.create(knex, item, '*')); - } - return (await Promise.all(items)).map((result) => result[0]); -} - -function searchEs(type, index, limit = 10) { - const executionQuery = new Search({ queryStringParameters: { limit } }, type, index); - return executionQuery.query(); -} - -test.before(async (t) => { - t.context.esIndices = []; - - await awsServices.s3().createBucket({ Bucket: process.env.system_bucket }); - await reconciliationReportModel.createTable(); - - const wKey = `${process.env.stackName}/workflows/${workflowList[0].name}.json`; - const tKey = `${process.env.stackName}/workflow_template.json`; - await Promise.all([ - promiseS3Upload({ - params: { - Bucket: process.env.system_bucket, - Key: wKey, - Body: JSON.stringify(workflowList[0]), - }, - }), - promiseS3Upload({ - params: { - Bucket: process.env.system_bucket, - Key: tKey, - Body: JSON.stringify({}), - }, - }), - ]); -}); - -test.beforeEach(async (t) => { - t.context.testDbName = `test_index_${cryptoRandomString({ length: 10 })}`; - const { knex, knexAdmin } = await generateLocalTestDb(t.context.testDbName, migrationDir); - t.context.knex = knex; - t.context.knexAdmin = knexAdmin; - t.context.esIndex = randomString(); - t.context.esAlias = randomString(); - await bootstrapElasticSearch({ - host: 'fakehost', - index: t.context.esIndex, - alias: t.context.esAlias, - }); - - t.context.esClient = new EsClient('fakehost'); - await t.context.esClient.initializeEsClient(); -}); - -test.afterEach.always(async (t) => { - const { esClient, esIndex, testDbName } = t.context; - await esClient.client.indices.delete({ index: esIndex }); - await destroyLocalTestDb({ - knex: t.context.knex, - knexAdmin: t.context.knexAdmin, - testDbName, - }); -}); - -test.after.always(async () => { - await recursivelyDeleteS3Bucket(process.env.system_bucket); -}); - -test('getEsRequestConcurrency respects concurrency value in payload', (t) => { - t.is(indexFromDatabase.getEsRequestConcurrency({ - esRequestConcurrency: 5, - }), 5); -}); - -test.serial('getEsRequestConcurrency respects ES_CONCURRENCY environment variable', (t) => { - process.env.ES_CONCURRENCY = 35; - t.is(indexFromDatabase.getEsRequestConcurrency({}), 35); - delete process.env.ES_CONCURRENCY; -}); - -test('getEsRequestConcurrency correctly returns 10 when nothing is specified', (t) => { - t.is(indexFromDatabase.getEsRequestConcurrency({}), 10); -}); - -test.serial('getEsRequestConcurrency throws an error when -1 is specified', (t) => { - t.throws( - () => indexFromDatabase.getEsRequestConcurrency({ - esRequestConcurrency: -1, - }), - { instanceOf: TypeError } - ); - - process.env.ES_CONCURRENCY = -1; - t.teardown(() => { - delete process.env.ES_CONCURRENCY; - }); - t.throws( - () => indexFromDatabase.getEsRequestConcurrency({}), - { instanceOf: TypeError } - ); -}); - -test.serial('getEsRequestConcurrency throws an error when "asdf" is specified', (t) => { - t.throws( - () => indexFromDatabase.getEsRequestConcurrency({ - esRequestConcurrency: 'asdf', - }), - { instanceOf: TypeError } - ); - - process.env.ES_CONCURRENCY = 'asdf'; - t.teardown(() => { - delete process.env.ES_CONCURRENCY; - }); - t.throws( - () => indexFromDatabase.getEsRequestConcurrency({}), - { instanceOf: TypeError } - ); -}); - -test.serial('getEsRequestConcurrency throws an error when 0 is specified', (t) => { - t.throws( - () => indexFromDatabase.getEsRequestConcurrency({ - esRequestConcurrency: 0, - }), - { instanceOf: TypeError } - ); - - process.env.ES_CONCURRENCY = 0; - t.teardown(() => { - delete process.env.ES_CONCURRENCY; - }); - t.throws( - () => indexFromDatabase.getEsRequestConcurrency({}), - { instanceOf: TypeError } - ); -}); - -test('No error is thrown if nothing is in the database', async (t) => { - const { esAlias, knex } = t.context; - - await t.notThrowsAsync(() => indexFromDatabase.indexFromDatabase({ - indexName: esAlias, - reconciliationReportsTable, - knex, - })); -}); - -test.serial('Lambda successfully indexes records of all types', async (t) => { - const knex = t.context.knex; - const { esAlias } = t.context; - - const numItems = 20; - - const fakeData = []; - const dateObject = { created_at: new Date(), updated_at: new Date() }; - const fakeCollectionRecords = await addFakeData( - knex, - numItems, - fakeCollectionRecordFactory, - new CollectionPgModel(), - dateObject - ); - fakeData.push(fakeCollectionRecords); - - const fakeExecutionRecords = await addFakeData( - knex, - numItems, - fakeExecutionRecordFactory, - new ExecutionPgModel(), - { ...dateObject } - ); - - const fakeGranuleRecords = await addFakeData( - knex, - numItems, - fakeGranuleRecordFactory, - new GranulePgModel(), - { collection_cumulus_id: fakeCollectionRecords[0].cumulus_id, ...dateObject } - ); - - const fakeProviderRecords = await addFakeData( - knex, - numItems, - fakeProviderRecordFactory, - new ProviderPgModel(), - dateObject - ); - - const fakePdrRecords = await addFakeData(knex, numItems, fakePdrRecordFactory, new PdrPgModel(), { - collection_cumulus_id: fakeCollectionRecords[0].cumulus_id, - provider_cumulus_id: fakeProviderRecords[0].cumulus_id, - ...dateObject, - }); - - const fakeReconciliationReportRecords = await addFakeDynamoData( - numItems, - fakeReconciliationReportFactory, - reconciliationReportModel - ); - - const fakeRuleRecords = await addFakeData( - knex, - numItems, - fakeRuleRecordFactory, - new RulePgModel(), - { - workflow: workflowList[0].name, - collection_cumulus_id: fakeCollectionRecords[0].cumulus_id, - provider_cumulus_id: fakeProviderRecords[0].cumulus_id, - ...dateObject, - } - ); - - await indexFromDatabase.handler({ - indexName: esAlias, - pageSize: 6, - knex, - }); - - const searchResults = await Promise.all([ - searchEs('collection', esAlias, '20'), - searchEs('execution', esAlias, '20'), - searchEs('granule', esAlias, '20'), - searchEs('pdr', esAlias, '20'), - searchEs('provider', esAlias, '20'), - searchEs('reconciliationReport', esAlias, '20'), - searchEs('rule', esAlias, '20'), - ]); - - searchResults.map((res) => t.is(res.meta.count, numItems)); - - const collectionResults = await Promise.all( - fakeCollectionRecords.map((r) => - translatePostgresCollectionToApiCollection(r)) - ); - const executionResults = await Promise.all( - fakeExecutionRecords.map((r) => translatePostgresExecutionToApiExecution(r)) - ); - const granuleResults = await Promise.all( - fakeGranuleRecords.map((r) => - translatePostgresGranuleToApiGranule({ - granulePgRecord: r, - knexOrTransaction: knex, - })) - ); - const pdrResults = await Promise.all( - fakePdrRecords.map((r) => translatePostgresPdrToApiPdr(r, knex)) - ); - const providerResults = await Promise.all( - fakeProviderRecords.map((r) => translatePostgresProviderToApiProvider(r)) - ); - const ruleResults = await Promise.all( - fakeRuleRecords.map((r) => translatePostgresRuleToApiRule(r, knex)) - ); - - t.deepEqual( - searchResults[0].results - .map((r) => omit(r, ['timestamp'])) - .sort((a, b) => (a.name > b.name ? 1 : -1)), - collectionResults - .sort((a, b) => (a.name > b.name ? 1 : -1)) - ); - - t.deepEqual( - sortAndFilter(searchResults[1].results, ['timestamp'], 'name'), - sortAndFilter(executionResults, ['timestamp'], 'name') - ); - - t.deepEqual( - sortAndFilter(searchResults[2].results, ['timestamp'], 'granuleId'), - sortAndFilter(granuleResults, ['timestamp'], 'granuleId') - ); - - t.deepEqual( - sortAndFilter(searchResults[3].results, ['timestamp'], 'pdrName'), - sortAndFilter(pdrResults, ['timestamp'], 'pdrName') - ); - - t.deepEqual( - sortAndFilter(searchResults[4].results, ['timestamp'], 'id'), - sortAndFilter(providerResults, ['timestamp'], 'id') - ); - - t.deepEqual( - sortAndFilter(searchResults[5].results, ['timestamp'], 'name'), - sortAndFilter(fakeReconciliationReportRecords, ['timestamp'], 'name') - ); - - t.deepEqual( - sortAndFilter(searchResults[6].results, ['timestamp'], 'name'), - sortAndFilter(ruleResults, ['timestamp'], 'name') - ); -}); - -test.serial('failure in indexing record of specific type should not prevent indexing of other records with same type', async (t) => { - const { esAlias, esClient, knex } = t.context; - const granulePgModel = new GranulePgModel(); - const numItems = 7; - const collectionRecord = await addFakeData( - knex, - 1, - fakeCollectionRecordFactory, - new CollectionPgModel() - ); - const fakeData = await addFakeData(knex, numItems, fakeGranuleRecordFactory, granulePgModel, { - collection_cumulus_id: collectionRecord[0].cumulus_id, - created_at: new Date(), - updated_at: new Date(), - }); - - let numCalls = 0; - const originalIndexGranule = indexer.indexGranule; - const successCount = 4; - const indexGranuleStub = sinon.stub(indexer, 'indexGranule') - .callsFake(( - esClientArg, - payload, - index - ) => { - numCalls += 1; - if (numCalls <= successCount) { - return originalIndexGranule(esClientArg, payload, index); - } - throw new Error('fake error'); - }); - - let searchResults; - try { - await indexFromDatabase.handler({ - indexName: esAlias, - reconciliationReportsTable, - knex, - }); - - searchResults = await searchEs('granule', esAlias); - - t.is(searchResults.meta.count, successCount); - - searchResults.results.forEach((result) => { - const sourceData = fakeData.find((data) => data.granule_id === result.granuleId); - const expected = { - collectionId: `${collectionRecord[0].name}___${collectionRecord[0].version}`, - granuleId: sourceData.granule_id, - status: sourceData.status, - }; - const actual = { - collectionId: result.collectionId, - granuleId: result.granuleId, - status: result.status, - }; - - t.deepEqual(expected, actual); - }); - } finally { - indexGranuleStub.restore(); - await Promise.all(fakeData.map( - // eslint-disable-next-line camelcase - ({ granule_id }) => granulePgModel.delete(knex, { granule_id }) - )); - await Promise.all(searchResults.results.map( - (result) => - esClient.client.delete({ - index: esAlias, - type: 'granule', - id: result.granuleId, - parent: result.collectionId, - refresh: true, - }) - )); - } -}); - -test.serial( - 'failure in indexing record of one type should not prevent indexing of other records with different type', - async (t) => { - const { esAlias, esClient, knex } = t.context; - const numItems = 2; - const collectionRecord = await addFakeData( - knex, - 1, - fakeCollectionRecordFactory, - new CollectionPgModel() - ); - const [fakeProviderData, fakeGranuleData] = await Promise.all([ - addFakeData( - knex, - numItems, - fakeProviderRecordFactory, - new ProviderPgModel() - ), - addFakeData( - knex, - numItems, - fakeGranuleRecordFactory, - new GranulePgModel(), - { collection_cumulus_id: collectionRecord[0].cumulus_id } - ), - ]); - - const indexGranuleStub = sinon - .stub(indexer, 'indexGranule') - .throws(new Error('error')); - - let searchResults; - try { - await indexFromDatabase.handler({ - indexName: esAlias, - reconciliationReportsTable, - knex, - }); - - searchResults = await searchEs('provider', esAlias); - - t.is(searchResults.meta.count, numItems); - - searchResults.results.forEach((result) => { - const sourceData = fakeProviderData.find( - (data) => data.name === result.id - ); - t.deepEqual( - { host: result.host, id: result.id, protocol: result.protocol }, - { - host: sourceData.host, - id: sourceData.name, - protocol: sourceData.protocol, - } - ); - }); - } finally { - indexGranuleStub.restore(); - await Promise.all( - fakeProviderData.map(({ name }) => { - const pgModel = new ProviderPgModel(); - return pgModel.delete(knex, { name }); - }) - ); - await Promise.all( - fakeGranuleData.map( - // eslint-disable-next-line camelcase - ({ granule_id }) => new GranulePgModel().delete(knex, { granule_id }) - ) - ); - await Promise.all( - searchResults.results.map((result) => - esClient.client.delete({ - index: esAlias, - type: 'provider', - id: result.id, - refresh: true, - })) - ); - } - } -); diff --git a/packages/api/tests/lambdas/test-internal-reconciliation-report.js b/packages/api/tests/lambdas/test-internal-reconciliation-report.js deleted file mode 100644 index e52473138b7..00000000000 --- a/packages/api/tests/lambdas/test-internal-reconciliation-report.js +++ /dev/null @@ -1,493 +0,0 @@ -'use strict'; - -const test = require('ava'); -const moment = require('moment'); -const flatten = require('lodash/flatten'); -const range = require('lodash/range'); -const cryptoRandomString = require('crypto-random-string'); - -const { - recursivelyDeleteS3Bucket, -} = require('@cumulus/aws-client/S3'); -const awsServices = require('@cumulus/aws-client/services'); -const { randomId } = require('@cumulus/common/test-utils'); -const { constructCollectionId, deconstructCollectionId } = require('@cumulus/message/Collections'); -const { generateGranuleApiRecord } = require('@cumulus/message/Granules'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const indexer = require('@cumulus/es-client/indexer'); -const { getEsClient } = require('@cumulus/es-client/search'); - -const { - CollectionPgModel, - destroyLocalTestDb, - generateLocalTestDb, - localStackConnectionEnv, - translateApiCollectionToPostgresCollection, - migrationDir, - translateApiGranuleToPostgresGranule, - GranulePgModel, - fakeProviderRecordFactory, - ProviderPgModel, - upsertGranuleWithExecutionJoinRecord, - fakeExecutionRecordFactory, - ExecutionPgModel, - FilePgModel, - translateApiFiletoPostgresFile, -} = require('@cumulus/db'); - -const { - fakeCollectionFactory, - // fakeFileFactory, - fakeGranuleFactoryV2, - fakeFileFactory, -} = require('../../lib/testUtils'); -const { - internalRecReportForCollections, - internalRecReportForGranules, -} = require('../../lambdas/internal-reconciliation-report'); -const { normalizeEvent } = require('../../lib/reconciliationReport/normalizeEvent'); -const models = require('../../models'); - -let esAlias; -let esIndex; -let esClient; - -test.before((t) => { - t.context.collectionPgModel = new CollectionPgModel(); - t.context.granulePgModel = new GranulePgModel(); - t.context.providerPgModel = new ProviderPgModel(); - t.context.executionPgModel = new ExecutionPgModel(); - t.context.filePgModel = new FilePgModel(); -}); - -test.beforeEach(async (t) => { - process.env.ReconciliationReportsTable = randomId('reconciliationTable'); - - t.context.bucketsToCleanup = []; - t.context.stackName = randomId('stack'); - t.context.systemBucket = randomId('bucket'); - process.env.system_bucket = t.context.systemBucket; - - await awsServices.s3().createBucket({ Bucket: t.context.systemBucket }) - .then(() => t.context.bucketsToCleanup.push(t.context.systemBucket)); - - await new models.ReconciliationReport().createTable(); - - esAlias = randomId('esalias'); - esIndex = randomId('esindex'); - process.env.ES_INDEX = esAlias; - await bootstrapElasticSearch({ - host: 'fakehost', - index: esIndex, - alias: esAlias, - }); - esClient = await getEsClient(); - - t.context.testDbName = `test_internal_recon_${cryptoRandomString({ length: 10 })}`; - const { knex, knexAdmin } = await generateLocalTestDb(t.context.testDbName, migrationDir); - t.context.knex = knex; - t.context.knexAdmin = knexAdmin; - process.env = { - ...process.env, - ...localStackConnectionEnv, - PG_DATABASE: t.context.testDbName, - }; -}); - -test.afterEach.always(async (t) => { - await destroyLocalTestDb({ - knex: t.context.knex, - knexAdmin: t.context.knexAdmin, - testDbName: t.context.testDbName, - }); - await Promise.all( - flatten([ - t.context.bucketsToCleanup.map(recursivelyDeleteS3Bucket), - new models.ReconciliationReport().deleteTable(), - ]) - ); - await esClient.client.indices.delete({ index: esIndex }); -}); - -test.serial('internalRecReportForCollections reports discrepancy of collection holdings in ES and DB', async (t) => { - const { knex, collectionPgModel } = t.context; - - const matchingColls = range(10).map(() => fakeCollectionFactory()); - const extraDbColls = range(2).map(() => fakeCollectionFactory()); - const extraEsColls = range(2).map(() => fakeCollectionFactory()); - - const conflictCollInDb = fakeCollectionFactory({ meta: { flag: 'db' } }); - const conflictCollInEs = { ...conflictCollInDb, meta: { flag: 'es' } }; - - const esCollections = matchingColls.concat(extraEsColls, conflictCollInEs); - const dbCollections = matchingColls.concat(extraDbColls, conflictCollInDb); - - await Promise.all( - esCollections.map((collection) => indexer.indexCollection(esClient, collection, esAlias)) - ); - - await Promise.all( - dbCollections.map((collection) => - collectionPgModel.create( - knex, - translateApiCollectionToPostgresCollection(collection) - )) - ); - - let report = await internalRecReportForCollections({}); - - t.is(report.okCount, 10); - t.is(report.onlyInEs.length, 2); - t.deepEqual(report.onlyInEs.sort(), - extraEsColls.map((coll) => constructCollectionId(coll.name, coll.version)).sort()); - t.is(report.onlyInDb.length, 2); - t.deepEqual(report.onlyInDb.sort(), - extraDbColls.map((coll) => constructCollectionId(coll.name, coll.version)).sort()); - t.is(report.withConflicts.length, 1); - t.deepEqual(report.withConflicts[0].es.collectionId, conflictCollInEs.collectionId); - t.deepEqual(report.withConflicts[0].db.collectionId, conflictCollInDb.collectionId); - - // start/end time include all the collections - const searchParams = { - startTimestamp: moment.utc().subtract(1, 'hour').format(), - endTimestamp: moment.utc().add(1, 'hour').format(), - }; - report = await internalRecReportForCollections(normalizeEvent(searchParams)); - t.is(report.okCount, 10); - t.is(report.onlyInEs.length, 2); - t.is(report.onlyInDb.length, 2); - t.is(report.withConflicts.length, 1); - - // start/end time has no matching collections - const paramsTimeOutOfRange = { - startTimestamp: moment.utc().add(1, 'hour').format(), - endTimestamp: moment.utc().add(2, 'hour').format(), - }; - - report = await internalRecReportForCollections(normalizeEvent(paramsTimeOutOfRange)); - t.is(report.okCount, 0); - t.is(report.onlyInEs.length, 0); - t.is(report.onlyInDb.length, 0); - t.is(report.withConflicts.length, 0); - - // collectionId matches the collection with conflicts - const collectionId = constructCollectionId(conflictCollInDb.name, conflictCollInDb.version); - const paramsCollectionId = { ...searchParams, collectionId: [collectionId, randomId('c')] }; - - report = await internalRecReportForCollections(normalizeEvent(paramsCollectionId)); - t.is(report.okCount, 0); - t.is(report.onlyInEs.length, 0); - t.is(report.onlyInDb.length, 0); - t.is(report.withConflicts.length, 1); -}); - -test.serial('internalRecReportForGranules reports discrepancy of granule holdings in ES and DB', async (t) => { - const { - knex, - collectionPgModel, - providerPgModel, - executionPgModel, - } = t.context; - - // Create collection in PG/ES - const collectionId = constructCollectionId(randomId('name'), randomId('version')); - - // Create provider in PG - const provider = fakeProviderRecordFactory(); - await providerPgModel.create(knex, provider); - - const matchingGrans = range(10).map(() => fakeGranuleFactoryV2({ - collectionId, - provider: provider.name, - })); - const additionalMatchingGrans = range(10).map(() => fakeGranuleFactoryV2({ - provider: provider.name, - })); - const extraDbGrans = range(2).map(() => fakeGranuleFactoryV2({ - collectionId, - provider: provider.name, - })); - const additionalExtraDbGrans = range(2).map(() => fakeGranuleFactoryV2()); - const extraEsGrans = range(2).map(() => fakeGranuleFactoryV2({ - provider: provider.name, - })); - const additionalExtraEsGrans = range(2) - .map(() => fakeGranuleFactoryV2({ - collectionId, - provider: provider.name, - })); - const conflictGranInDb = fakeGranuleFactoryV2({ collectionId, status: 'completed' }); - const conflictGranInEs = { ...conflictGranInDb, status: 'failed' }; - - const esGranules = matchingGrans - .concat(additionalMatchingGrans, extraEsGrans, additionalExtraEsGrans, conflictGranInEs); - const dbGranules = matchingGrans - .concat(additionalMatchingGrans, extraDbGrans, additionalExtraDbGrans, conflictGranInDb); - - // add granules and related collections to es and db - await Promise.all( - esGranules.map(async (granule) => { - const collection = fakeCollectionFactory({ - ...deconstructCollectionId(granule.collectionId), - }); - await indexer.indexCollection(esClient, collection, esAlias); - await collectionPgModel.upsert( - knex, - translateApiCollectionToPostgresCollection(collection) - ); - await indexer.indexGranule(esClient, granule, esAlias); - }) - ); - - await Promise.all( - dbGranules.map(async (granule) => { - const pgGranule = await translateApiGranuleToPostgresGranule({ - dynamoRecord: granule, - knexOrTransaction: knex, - }); - let pgExecution = {}; - if (granule.execution) { - const pgExecutionData = fakeExecutionRecordFactory({ - url: granule.execution, - }); - ([pgExecution] = await executionPgModel.create(knex, pgExecutionData)); - } - return upsertGranuleWithExecutionJoinRecord({ - executionCumulusId: pgExecution.cumulus_id, - granule: pgGranule, - knexTransaction: knex, - }); - }) - ); - - let report = await internalRecReportForGranules({ knex }); - t.is(report.okCount, 20); - t.is(report.onlyInEs.length, 4); - t.deepEqual(report.onlyInEs.map((gran) => gran.granuleId).sort(), - extraEsGrans.concat(additionalExtraEsGrans).map((gran) => gran.granuleId).sort()); - t.is(report.onlyInDb.length, 4); - t.deepEqual(report.onlyInDb.map((gran) => gran.granuleId).sort(), - extraDbGrans.concat(additionalExtraDbGrans).map((gran) => gran.granuleId).sort()); - t.is(report.withConflicts.length, 1); - t.deepEqual(report.withConflicts[0].es.granuleId, conflictGranInEs.granuleId); - t.deepEqual(report.withConflicts[0].db.granuleId, conflictGranInDb.granuleId); - - // start/end time include all the collections and granules - const searchParams = { - reportType: 'Internal', - startTimestamp: moment.utc().subtract(1, 'hour').format(), - endTimestamp: moment.utc().add(1, 'hour').format(), - }; - report = await internalRecReportForGranules({ - ...normalizeEvent(searchParams), - knex, - }); - t.is(report.okCount, 20); - t.is(report.onlyInEs.length, 4); - t.is(report.onlyInDb.length, 4); - t.is(report.withConflicts.length, 1); - - // start/end time has no matching collections and granules - const outOfRangeParams = { - startTimestamp: moment.utc().add(1, 'hour').format(), - endTimestamp: moment.utc().add(2, 'hour').format(), - }; - - report = await internalRecReportForGranules({ - ...normalizeEvent(outOfRangeParams), - knex, - }); - t.is(report.okCount, 0); - t.is(report.onlyInEs.length, 0); - t.is(report.onlyInDb.length, 0); - t.is(report.withConflicts.length, 0); - - // collectionId, provider parameters - const collectionProviderParams = { ...searchParams, collectionId, provider: provider.name }; - report = await internalRecReportForGranules({ - ...normalizeEvent(collectionProviderParams), - knex, - }); - t.is(report.okCount, 10); - t.is(report.onlyInEs.length, 2); - t.deepEqual(report.onlyInEs.map((gran) => gran.granuleId).sort(), - additionalExtraEsGrans.map((gran) => gran.granuleId).sort()); - t.is(report.onlyInDb.length, 2); - t.deepEqual(report.onlyInDb.map((gran) => gran.granuleId).sort(), - extraDbGrans.map((gran) => gran.granuleId).sort()); - t.is(report.withConflicts.length, 0); - - // provider parameter - const providerParams = { ...searchParams, provider: [randomId('p'), provider.name] }; - report = await internalRecReportForGranules({ - ...normalizeEvent(providerParams), - knex, - }); - t.is(report.okCount, 20); - t.is(report.onlyInEs.length, 4); - t.deepEqual(report.onlyInEs.map((gran) => gran.granuleId).sort(), - extraEsGrans.concat(additionalExtraEsGrans).map((gran) => gran.granuleId).sort()); - t.is(report.onlyInDb.length, 2); - t.deepEqual(report.onlyInDb.map((gran) => gran.granuleId).sort(), - extraDbGrans.map((gran) => gran.granuleId).sort()); - t.is(report.withConflicts.length, 0); - - // collectionId, granuleId parameters - const granuleId = conflictGranInDb.granuleId; - const granuleIdParams = { - ...searchParams, - granuleId: [granuleId, extraEsGrans[0].granuleId, randomId('g')], - collectionId: [collectionId, extraEsGrans[0].collectionId, extraEsGrans[1].collectionId], - }; - report = await internalRecReportForGranules({ - ...normalizeEvent(granuleIdParams), - knex, - }); - t.is(report.okCount, 0); - t.is(report.onlyInEs.length, 1); - t.is(report.onlyInEs[0].granuleId, extraEsGrans[0].granuleId); - t.is(report.onlyInDb.length, 0); - t.is(report.withConflicts.length, 1); -}); - -test.serial('internalRecReportForGranules handles generated granules with custom timestamps', async (t) => { - const { - knex, - collectionPgModel, - providerPgModel, - executionPgModel, - } = t.context; - - // Create collection in PG/ES - const collectionId = constructCollectionId(randomId('name'), randomId('version')); - const collection = fakeCollectionFactory({ - ...deconstructCollectionId(collectionId), - }); - await indexer.indexCollection(esClient, collection, esAlias); - await collectionPgModel.upsert( - knex, - translateApiCollectionToPostgresCollection(collection) - ); - - // Create provider in PG - const provider = fakeProviderRecordFactory(); - await providerPgModel.create(knex, provider); - - // Use date string with extra precision to make sure it is saved - // correctly in dynamo, PG, an Elasticsearch - const dateString = '2018-04-25T21:45:45.524053'; - - await Promise.all(range(5).map(async () => { - const fakeGranule = fakeGranuleFactoryV2({ - collectionId, - provider: provider.name, - }); - - const processingTimeInfo = { - processingStartDateTime: dateString, - processingEndDateTime: dateString, - }; - - const cmrTemporalInfo = { - beginningDateTime: dateString, - endingDateTime: dateString, - productionDateTime: dateString, - lastUpdateDateTime: dateString, - }; - - const apiGranule = await generateGranuleApiRecord({ - ...fakeGranule, - granule: fakeGranule, - executionUrl: fakeGranule.execution, - processingTimeInfo, - cmrTemporalInfo, - }); - const pgGranule = await translateApiGranuleToPostgresGranule({ - dynamoRecord: apiGranule, - knexOrTransaction: knex, - }); - - let pgExecution = {}; - if (apiGranule.execution) { - const pgExecutionData = fakeExecutionRecordFactory({ - url: apiGranule.execution, - }); - ([pgExecution] = await executionPgModel.create(knex, pgExecutionData)); - } - await upsertGranuleWithExecutionJoinRecord({ - executionCumulusId: pgExecution.cumulus_id, - granule: pgGranule, - knexTransaction: knex, - }); - await indexer.indexGranule(esClient, apiGranule, esAlias); - })); - - const report = await internalRecReportForGranules({ knex }); - t.is(report.okCount, 5); - t.is(report.onlyInEs.length, 0); - t.is(report.onlyInDb.length, 0); -}); - -test.serial('internalRecReportForGranules handles granules with files', async (t) => { - const { - knex, - collectionPgModel, - executionPgModel, - filePgModel, - } = t.context; - - // Create collection in PG/ES - const collectionId = constructCollectionId(randomId('name'), randomId('version')); - const collection = fakeCollectionFactory({ - ...deconstructCollectionId(collectionId), - }); - await indexer.indexCollection(esClient, collection, esAlias); - await collectionPgModel.upsert( - knex, - translateApiCollectionToPostgresCollection(collection) - ); - await Promise.all(range(2).map(async () => { - const fakeGranule = fakeGranuleFactoryV2({ - collectionId, - files: [fakeFileFactory(), fakeFileFactory(), fakeFileFactory()], - }); - - const fakeCmrUtils = { - getGranuleTemporalInfo: () => Promise.resolve({}), - }; - const apiGranule = await generateGranuleApiRecord({ - ...fakeGranule, - granule: fakeGranule, - executionUrl: fakeGranule.execution, - cmrUtils: fakeCmrUtils, - }); - const pgGranule = await translateApiGranuleToPostgresGranule({ - dynamoRecord: apiGranule, - knexOrTransaction: knex, - }); - - const pgExecutionData = fakeExecutionRecordFactory({ - url: apiGranule.execution, - }); - const [pgExecution] = await executionPgModel.create(knex, pgExecutionData); - - const [pgGranuleRecord] = await upsertGranuleWithExecutionJoinRecord({ - executionCumulusId: pgExecution.cumulus_id, - granule: pgGranule, - knexTransaction: knex, - }); - await Promise.all(apiGranule.files.map(async (file) => { - const pgFile = translateApiFiletoPostgresFile(file); - await filePgModel.create(knex, { - ...pgFile, - granule_cumulus_id: pgGranuleRecord.cumulus_id, - }); - })); - await indexer.indexGranule(esClient, apiGranule, esAlias); - })); - - const report = await internalRecReportForGranules({ knex }); - t.is(report.okCount, 2); - t.is(report.onlyInEs.length, 0); - t.is(report.onlyInDb.length, 0); -}); diff --git a/packages/api/tests/lambdas/test-orca-backup-reconciliation-report.js b/packages/api/tests/lambdas/test-orca-backup-reconciliation-report.js index c2cad6d78ae..3a00af398a4 100644 --- a/packages/api/tests/lambdas/test-orca-backup-reconciliation-report.js +++ b/packages/api/tests/lambdas/test-orca-backup-reconciliation-report.js @@ -4,11 +4,27 @@ const test = require('ava'); const rewire = require('rewire'); const sinon = require('sinon'); const sortBy = require('lodash/sortBy'); +const omit = require('lodash/omit'); +const cryptoRandomString = require('crypto-random-string'); +// TODO abstract this setup const { randomId } = require('@cumulus/common/test-utils'); -const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); -const indexer = require('@cumulus/es-client/indexer'); -const { getEsClient } = require('@cumulus/es-client/search'); +const { deconstructCollectionId } = require('@cumulus/message/Collections'); +const { + fakeProviderRecordFactory, + CollectionPgModel, + GranulePgModel, + FilePgModel, + GranulesExecutionsPgModel, + ProviderPgModel, + migrationDir, + destroyLocalTestDb, + generateLocalTestDb, + translateApiGranuleToPostgresGranule, + translateApiCollectionToPostgresCollection, + localStackConnectionEnv, + translateApiFiletoPostgresFile, +} = require('@cumulus/db'); const { fakeCollectionFactory, @@ -24,9 +40,19 @@ const ORCASearchCatalogQueue = require('../../lib/ORCASearchCatalogQueue'); const shouldFileBeExcludedFromOrca = OBRP.__get__('shouldFileBeExcludedFromOrca'); const getReportForOneGranule = OBRP.__get__('getReportForOneGranule'); -let esAlias; -let esIndex; -let esClient; +function translateTestGranuleObject(apiGranule) { + const { name: collectionName, version: collectionVersion } = + deconstructCollectionId(apiGranule.collectionId); + const ProviderName = apiGranule.provider; + return { + ...(omit(apiGranule, ['collectionId', 'provider', 'createdAt', 'updatedAt'])), + collectionName, + collectionVersion, + ProviderName, + created_at: new Date(apiGranule.createdAt), + updated_at: new Date(apiGranule.updatedAt), + }; +} function fakeCollectionsAndGranules() { const fakeCollectionV1 = fakeCollectionFactory({ @@ -73,7 +99,7 @@ function fakeCollectionsAndGranules() { ], }; - // granule is in cumulus only, should not be in orca, and conform to configuratio + // granule is in cumulus only, should not be in orca, and conform to configuration const matchingCumulusOnlyGran = { ...fakeGranuleFactoryV2(), granuleId: randomId('matchingCumulusOnlyGranId'), @@ -82,12 +108,12 @@ function fakeCollectionsAndGranules() { { bucket: 'cumulus-protected-bucket', fileName: 'fakeFileName.xml', - key: 'fakePath/fakeFileName.xml', + key: 'fakePath/fakeFileName4.xml', }, { bucket: 'cumulus-protected-bucket', fileName: 'fakeFileName.hdf.met', - key: 'fakePath/fakeFileName.hdf.met', + key: 'fakePath/fakeFileName4.hdf.met', }, ], }; @@ -102,22 +128,22 @@ function fakeCollectionsAndGranules() { { bucket: 'cumulus-protected-bucket', fileName: 'fakeFileName.hdf', - key: 'fakePath/fakeFileName.hdf', + key: 'fakePath/fakeFileName3.hdf', }, { bucket: 'cumulus-private-bucket', fileName: 'fakeFileName.hdf.met', - key: 'fakePath/fakeFileName.hdf.met', + key: 'fakePath/fakeFileName3.hdf.met', }, { bucket: 'cumulus-fake-bucket', fileName: 'fakeFileName_onlyInCumulus.jpg', - key: 'fakePath/fakeFileName_onlyInCumulus.jpg', + key: 'fakePath/fakeFileName3_onlyInCumulus.jpg', }, { bucket: 'cumulus-fake-bucket-2', fileName: 'fakeFileName.cmr.xml', - key: 'fakePath/fakeFileName.cmr.xml', + key: 'fakePath/fakeFileName3.cmr.xml', }, ], }; @@ -131,19 +157,19 @@ function fakeCollectionsAndGranules() { name: 'fakeFileName.hdf', cumulusArchiveLocation: 'cumulus-protected-bucket', orcaArchiveLocation: 'orca-bucket', - keyPath: 'fakePath/fakeFileName.hdf', + keyPath: 'fakePath/fakeFileName3.hdf', }, { name: 'fakeFileName_onlyInOrca.jpg', cumulusArchiveLocation: 'cumulus-fake-bucket', orcaArchiveLocation: 'orca-bucket', - keyPath: 'fakePath/fakeFileName_onlyInOrca.jpg', + keyPath: 'fakePath/fakeFileName3_onlyInOrca.jpg', }, { name: 'fakeFileName.cmr.xml', cumulusArchiveLocation: 'cumulus-fake-bucket-2', orcaArchiveLocation: 'orca-bucket', - keyPath: 'fakePath/fakeFileName.cmr.xml', + keyPath: 'fakePath/fakeFileName3.cmr.xml', }, ], }; @@ -191,19 +217,34 @@ test.beforeEach(async (t) => { t.context.systemBucket = randomId('bucket'); process.env.system_bucket = t.context.systemBucket; - esAlias = randomId('esalias'); - esIndex = randomId('esindex'); - process.env.ES_INDEX = esAlias; - await bootstrapElasticSearch({ - host: 'fakehost', - index: esIndex, - alias: esAlias, - }); - esClient = await getEsClient(); + // Setup Postgres DB + + t.context.testDbName = `orca_backup_recon_${cryptoRandomString({ length: 10 })}`; + const { knexAdmin, knex } = await generateLocalTestDb( + t.context.testDbName, + migrationDir, + { dbMaxPool: 10 } + ); + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + t.context.granulePgModel = new GranulePgModel(); + t.context.collectionPgModel = new CollectionPgModel(); + t.context.granulesExecutionsPgModel = new GranulesExecutionsPgModel(); + t.context.filePgModel = new FilePgModel(); + + process.env = { + ...process.env, + ...localStackConnectionEnv, + PG_DATABASE: t.context.testDbName, + dbMaxPool: 10, + }; }); -test.afterEach.always(async () => { - await esClient.client.indices.delete({ index: esIndex }); +test.afterEach.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + }); }); test.serial('shouldFileBeExcludedFromOrca returns true for configured file types', (t) => { @@ -224,13 +265,19 @@ test.serial('shouldFileBeExcludedFromOrca returns true for configured file types t.false(shouldFileBeExcludedFromOrca(collectionsConfig, `${randomId('coll')}`, randomId('file'))); }); -test.serial('getReportForOneGranule reports ok for one granule in both cumulus and orca with no file discrepancy', (t) => { +test.serial('getReportForOneGranule reports ok for one granule in both cumulus and orca with no file discrepancy', async (t) => { + const { knex } = t.context; const collectionsConfig = {}; const { matchingCumulusGran: cumulusGranule, matchingOrcaGran: orcaGranule, } = fakeCollectionsAndGranules(); - const report = getReportForOneGranule({ collectionsConfig, cumulusGranule, orcaGranule }); + const report = await getReportForOneGranule({ + collectionsConfig, + cumulusGranule: translateTestGranuleObject(cumulusGranule), + orcaGranule, + knex, + }); t.true(report.ok); t.is(report.okFilesCount, 1); t.is(report.cumulusFilesCount, 1); @@ -238,7 +285,9 @@ test.serial('getReportForOneGranule reports ok for one granule in both cumulus a t.is(report.conflictFiles.length, 0); }); -test.serial('getReportForOneGranule reports no ok for one granule in both cumulus and orca with file discrepancy', (t) => { +test.serial('getReportForOneGranule reports no ok for one granule in both cumulus and orca with file discrepancy', async (t) => { + const { knex } = t.context; + const collectionsConfig = { fakeCollection___v1: { orca: { @@ -246,11 +295,17 @@ test.serial('getReportForOneGranule reports no ok for one granule in both cumulu }, }, }; - const { - conflictCumulusGran: cumulusGranule, - conflictOrcaGran: orcaGranule, - } = fakeCollectionsAndGranules(); - const report = getReportForOneGranule({ collectionsConfig, cumulusGranule, orcaGranule }); + + const granules = fakeCollectionsAndGranules(); + const cumulusGranule = translateTestGranuleObject(granules.conflictCumulusGran); + const orcaGranule = granules.conflictOrcaGran; + + const report = await getReportForOneGranule({ + collectionsConfig, + cumulusGranule, + orcaGranule, + knex, + }); t.false(report.ok); t.is(report.okFilesCount, 2); t.is(report.cumulusFilesCount, 4); @@ -281,10 +336,14 @@ test.serial('getReportForOneGranule reports ok for one granule in cumulus only w }, }, }; - const { - matchingCumulusOnlyGran: cumulusGranule, - } = fakeCollectionsAndGranules(); - const report = getReportForOneGranule({ collectionsConfig, cumulusGranule }); + + const granules = fakeCollectionsAndGranules(); + const cumulusGranule = translateTestGranuleObject(granules.matchingCumulusOnlyGran); + + const report = getReportForOneGranule({ + collectionsConfig, + cumulusGranule, + }); t.true(report.ok); t.is(report.okFilesCount, 2); t.is(report.cumulusFilesCount, 2); @@ -300,10 +359,14 @@ test.serial('getReportForOneGranule reports not ok for one granule in cumulus on }, }, }; - const { - conflictCumulusOnlyGran: cumulusGranule, - } = fakeCollectionsAndGranules(); - const report = getReportForOneGranule({ collectionsConfig, cumulusGranule }); + + const granules = fakeCollectionsAndGranules(); + const cumulusGranule = translateTestGranuleObject(granules.conflictCumulusOnlyGran); + + const report = getReportForOneGranule({ + collectionsConfig, + cumulusGranule, + }); t.false(report.ok); t.is(report.okFilesCount, 1); t.is(report.cumulusFilesCount, 2); @@ -313,10 +376,14 @@ test.serial('getReportForOneGranule reports not ok for one granule in cumulus on test.serial('getReportForOneGranule reports ok for one granule in cumulus only with no files', (t) => { const collectionsConfig = {}; - const { - cumulusOnlyGranNoFile: cumulusGranule, - } = fakeCollectionsAndGranules(); - const report = getReportForOneGranule({ collectionsConfig, cumulusGranule }); + + const granules = fakeCollectionsAndGranules(); + const cumulusGranule = translateTestGranuleObject(granules.cumulusOnlyGranNoFile); + + const report = getReportForOneGranule({ + collectionsConfig, + cumulusGranule, + }); t.true(report.ok); t.is(report.okFilesCount, 0); t.is(report.cumulusFilesCount, 0); @@ -325,6 +392,7 @@ test.serial('getReportForOneGranule reports ok for one granule in cumulus only w }); test.serial('orcaReconciliationReportForGranules reports discrepancy of granule holdings in cumulus and orca', async (t) => { + const { collectionPgModel, granulePgModel, filePgModel, knex } = t.context; const { fakeCollectionV1, fakeCollectionV2, @@ -338,24 +406,49 @@ test.serial('orcaReconciliationReportForGranules reports discrepancy of granule conflictCumulusOnlyGran, } = fakeCollectionsAndGranules(); - const esGranules = [ + // Create provider + const fakeProvider = fakeProviderRecordFactory({ name: 'fakeProvider' }); + const fakeProvider2 = fakeProviderRecordFactory({ name: 'fakeProvider2' }); + const providerPgModel = new ProviderPgModel(); + await Promise.all( + [fakeProvider, fakeProvider2].map((p) => + providerPgModel.create(knex, p)) + ); + + // Create collections + const pgCollections = await Promise.all( + [fakeCollectionV1, fakeCollectionV2].map((c) => translateApiCollectionToPostgresCollection(c)) + ); + await Promise.all( + pgCollections.map((collection) => collectionPgModel.create(knex, collection)) + ); + + const apiGranules = [ cumulusOnlyGranNoFile, conflictCumulusGran, matchingCumulusGran, matchingCumulusOnlyGran, conflictCumulusOnlyGran, ]; - const esCollections = [fakeCollectionV1, fakeCollectionV2]; - // add granules and related collections to es and db - await Promise.all( - esCollections.map(async (collection) => { - await indexer.indexCollection(esClient, collection, esAlias); - }) - ); + // Create granules await Promise.all( - esGranules.map(async (granule) => { - await indexer.indexGranule(esClient, granule, esAlias); + apiGranules.map(async (granule) => { + const pgGranule = await translateApiGranuleToPostgresGranule({ + dynamoRecord: granule, + knexOrTransaction: knex, + }); + const pgRecord = await granulePgModel.create(knex, pgGranule); + if (!granule.files) { + return; + } + const pgFiles = granule.files.map((f) => (translateApiFiletoPostgresFile(f))); + await Promise.all( + pgFiles.map(async (f) => await filePgModel.create(knex, { + ...f, + granule_cumulus_id: pgRecord[0].cumulus_id, + })) + ); }) ); diff --git a/packages/api/tests/lib/reconciliationReport/test-normalizeEvent.js b/packages/api/tests/lib/reconciliationReport/test-normalizeEvent.js index c71989788f9..b197e6e016a 100644 --- a/packages/api/tests/lib/reconciliationReport/test-normalizeEvent.js +++ b/packages/api/tests/lib/reconciliationReport/test-normalizeEvent.js @@ -1,6 +1,6 @@ const test = require('ava'); const omit = require('lodash/omit'); -const { InvalidArgument } = require('@cumulus/errors'); +const { InvalidArgument, MissingRequiredArgument } = require('@cumulus/errors'); const { constructCollectionId } = require('@cumulus/message/Collections'); const { randomId } = require('@cumulus/common/test-utils'); const { normalizeEvent } = require('../../../lib/reconciliationReport/normalizeEvent'); @@ -209,7 +209,7 @@ test('normalizeEvent throws error if provider and granuleId are passed to non-In test('Invalid report type throws InvalidArgument error', (t) => { const reportType = randomId('badType'); - const inputEvent = { reportType }; + const inputEvent = { reportType, systemBucket: 'systemBucket', stackName: 'stackName' }; t.throws(() => normalizeEvent(inputEvent), { instanceOf: InvalidArgument, @@ -220,6 +220,32 @@ test('Invalid report type throws InvalidArgument error', (t) => { test('valid Reports types from reconciliation schema do not throw an error.', (t) => { const validReportTypes = reconciliationReport.properties.type.enum; validReportTypes.forEach((reportType) => { - t.notThrows(() => normalizeEvent({ reportType })); + t.notThrows(() => normalizeEvent({ reportType, systemBucket: 'systemBucket', stackName: 'stackName' })); + }); +}); + +test('normalizeEvent throws error if no systemBucket is provided', (t) => { + const inputEvent = { + endTimestamp: new Date().toISOString(), + reportType: 'Inventory', + stackName: 'stackName', + startTimestamp: new Date().toISOString(), + }; + t.throws(() => normalizeEvent(inputEvent), { + instanceOf: MissingRequiredArgument, + message: 'systemBucket is required.', + }); +}); + +test('normalizeEvent throws error if no stackName is provided', (t) => { + const inputEvent = { + endTimestamp: new Date().toISOString(), + reportType: 'Inventory', + startTimestamp: new Date().toISOString(), + systemBucket: 'systemBucket', + }; + t.throws(() => normalizeEvent(inputEvent), { + instanceOf: MissingRequiredArgument, + message: 'stackName is required.', }); }); diff --git a/packages/api/tests/lib/test-ingest.js b/packages/api/tests/lib/test-ingest.js index 521948ab899..2673f3f4756 100644 --- a/packages/api/tests/lib/test-ingest.js +++ b/packages/api/tests/lib/test-ingest.js @@ -22,9 +22,6 @@ const { fakeCollectionRecordFactory, getUniqueGranuleByGranuleId, } = require('@cumulus/db'); -const { - createTestIndex, -} = require('@cumulus/es-client/testUtils'); const { fakeGranuleFactoryV2, fakeCollectionFactory, @@ -37,11 +34,6 @@ const { const testDbName = randomString(12); const sandbox = sinon.createSandbox(); -const FakeEsClient = sandbox.stub(); -const esSearchStub = sandbox.stub(); -const esScrollStub = sandbox.stub(); -FakeEsClient.prototype.scroll = esScrollStub; -FakeEsClient.prototype.search = esSearchStub; let fakeExecution; let testCumulusMessage; @@ -64,10 +56,6 @@ test.before(async (t) => { t.context.knexAdmin = knexAdmin; t.context.granuleId = randomString(); - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - const { TopicArn } = await createSnsTopic(randomString()); t.context.granules_sns_topic_arn = TopicArn; process.env.granule_sns_topic_arn = t.context.granules_sns_topic_arn; diff --git a/packages/api/tests/lib/test-reconciliationReport.js b/packages/api/tests/lib/test-reconciliationReport.js index 3722a3bdb11..e707df916de 100644 --- a/packages/api/tests/lib/test-reconciliationReport.js +++ b/packages/api/tests/lib/test-reconciliationReport.js @@ -1,5 +1,4 @@ const test = require('ava'); -const cryptoRandomString = require('crypto-random-string'); const rewire = require('rewire'); const range = require('lodash/range'); @@ -8,13 +7,8 @@ const { constructCollectionId } = require('@cumulus/message/Collections'); const sortBy = require('lodash/sortBy'); const { convertToDBCollectionSearchObject, - convertToESCollectionSearchParams, - convertToESGranuleSearchParams, - convertToESGranuleSearchParamsWithCreatedAtRange, convertToOrcaGranuleSearchParams, filterDBCollections, - searchParamsForCollectionIdArray, - compareEsGranuleAndApiGranule, } = require('../../lib/reconciliationReport'); const { fakeCollectionFactory } = require('../../lib/testUtils'); @@ -37,63 +31,6 @@ test('dateToValue returns undefined for any string that cannot be converted to a testStrings.map((testVal) => t.is(dateToValue(testVal), undefined)); }); -test('convertToESCollectionSearchParams returns correct search object.', (t) => { - const startTimestamp = '2000-10-31T15:00:00.000Z'; - const endTimestamp = '2001-10-31T15:00:00.000Z'; - const testObj = { - startTimestamp, - endTimestamp, - anotherKey: 'anything', - anotherKey2: 'they are ignored', - }; - - const expected = { - updatedAt__from: 973004400000, - updatedAt__to: 1004540400000, - }; - - const actual = convertToESCollectionSearchParams(testObj); - t.deepEqual(actual, expected); -}); - -test('convertToESGranuleSearchParams returns correct search object.', (t) => { - const startTimestamp = '2010-01-01T00:00:00.000Z'; - const endTimestamp = '2011-10-01T12:00:00.000Z'; - const testObj = { - startTimestamp, - endTimestamp, - anotherKey: 'anything', - anotherKey2: 'they are ignored', - }; - - const expected = { - updatedAt__from: 1262304000000, - updatedAt__to: 1317470400000, - }; - - const actual = convertToESGranuleSearchParams(testObj); - t.deepEqual(actual, expected); -}); - -test('convertToESGranuleSearchParamsWithCreatedAtRange returns correct search object.', (t) => { - const startTimestamp = '2010-01-01T00:00:00.000Z'; - const endTimestamp = '2011-10-01T12:00:00.000Z'; - const testObj = { - startTimestamp, - endTimestamp, - anotherKey: 'anything', - anotherKey2: 'they are ignored', - }; - - const expected = { - createdAt__from: 1262304000000, - createdAt__to: 1317470400000, - }; - - const actual = convertToESGranuleSearchParamsWithCreatedAtRange(testObj); - t.deepEqual(actual, expected); -}); - test('convertToOrcaGranuleSearchParams returns correct search object.', (t) => { const startTimestamp = '2010-01-01T00:00:00.000Z'; const endTimestamp = '2011-10-01T12:00:00.000Z'; @@ -119,28 +56,6 @@ test('convertToOrcaGranuleSearchParams returns correct search object.', (t) => { t.deepEqual(actual, expected); }); -test('convertToESCollectionSearchParams returns correct search object with collectionIds.', (t) => { - const startTimestamp = '2000-10-31T15:00:00.000Z'; - const endTimestamp = '2001-10-31T15:00:00.000Z'; - const collectionIds = ['name____version', 'name2___version']; - const testObj = { - startTimestamp, - endTimestamp, - collectionIds, - anotherKey: 'anything', - anotherKey2: 'they are ignored', - }; - - const expected = { - updatedAt__from: 973004400000, - updatedAt__to: 1004540400000, - _id__in: 'name____version,name2___version', - }; - - const actual = convertToESCollectionSearchParams(testObj); - t.deepEqual(actual, expected); -}); - test('convertToDBCollectionSearchParams returns correct search object with collectionIds.', (t) => { const startTimestamp = '2000-10-31T15:00:00.000Z'; const endTimestamp = '2001-10-31T15:00:00.000Z'; @@ -229,101 +144,3 @@ test("filterDBCollections filters collections by recReportParams's collectionIds t.deepEqual(actual, expected); }); - -test('searchParamsForCollectionIdArray converts array of collectionIds to a proper object to pass to the query command.', (t) => { - const collectionIds = ['col1___ver1', 'col1___ver2', 'col2___ver1']; - - const expectedInputQueryParams = { - _id__in: 'col1___ver1,col1___ver2,col2___ver1', - }; - - const actualSearchParams = searchParamsForCollectionIdArray(collectionIds); - t.deepEqual(actualSearchParams, expectedInputQueryParams); -}); - -test('compareEsGranuleAndApiGranule returns true for matching granules', (t) => { - const granule = { - granuleId: cryptoRandomString({ length: 5 }), - }; - const granule2 = { ...granule, files: [] }; - t.true(compareEsGranuleAndApiGranule(granule, granule2)); -}); - -test('compareEsGranuleAndApiGranule returns false for granules with different values', (t) => { - const granule = { - granuleId: cryptoRandomString({ length: 5 }), - }; - const granule2 = { ...granule, foo: 'bar' }; - t.false(compareEsGranuleAndApiGranule(granule, granule2)); -}); - -test('compareEsGranuleAndApiGranule returns false if one granule has files and other does not', (t) => { - const granule = { - granuleId: cryptoRandomString({ length: 5 }), - }; - const granule2 = { - ...granule, - files: [{ - bucket: 'bucket', - key: 'key', - }], - }; - t.false(compareEsGranuleAndApiGranule(granule, granule2)); -}); - -test('compareEsGranuleAndApiGranule returns false if granule file is missing from second granule', (t) => { - const granule = { - granuleId: cryptoRandomString({ length: 5 }), - files: [{ - bucket: 'bucket', - key: 'key', - }], - }; - const granule2 = { - ...granule, - files: [{ - bucket: 'bucket', - key: 'key2', - }], - }; - t.false(compareEsGranuleAndApiGranule(granule, granule2)); -}); - -test('compareEsGranuleAndApiGranule returns false if granule files have different properties', (t) => { - const granule = { - granuleId: cryptoRandomString({ length: 5 }), - files: [{ - bucket: 'bucket', - key: 'key', - }], - }; - const granule2 = { - ...granule, - files: [{ - bucket: 'bucket', - key: 'key', - size: 5, - }], - }; - t.false(compareEsGranuleAndApiGranule(granule, granule2)); -}); - -test('compareEsGranuleAndApiGranule returns false if granule files have different values for same property', (t) => { - const granule = { - granuleId: cryptoRandomString({ length: 5 }), - files: [{ - bucket: 'bucket', - key: 'key', - size: 1, - }], - }; - const granule2 = { - ...granule, - files: [{ - bucket: 'bucket', - key: 'key', - size: 5, - }], - }; - t.false(compareEsGranuleAndApiGranule(granule, granule2)); -}); diff --git a/packages/api/tests/performance/lib/test-write-granules.js b/packages/api/tests/performance/lib/test-write-granules.js index 676d7d92b70..125f9194abd 100644 --- a/packages/api/tests/performance/lib/test-write-granules.js +++ b/packages/api/tests/performance/lib/test-write-granules.js @@ -6,10 +6,6 @@ const pSettle = require('p-settle'); const cryptoRandomString = require('crypto-random-string'); const cloneDeep = require('lodash/cloneDeep'); -const { - getEsClient, - Search, -} = require('@cumulus/es-client/search'); const { createSnsTopic } = require('@cumulus/aws-client/SNS'); const StepFunctions = require('@cumulus/aws-client/StepFunctions'); @@ -39,10 +35,6 @@ const { const { getExecutionUrlFromArn, } = require('@cumulus/message/Executions'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { writeGranulesFromMessage, @@ -71,15 +63,6 @@ test.before(async (t) => { t.context.knex = knex; console.log(`Test DB max connection pool: ${t.context.knex.client.pool.max}`); - - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esGranulesClient = new Search( - {}, - 'granule', - t.context.esIndex - ); }); test.beforeEach(async (t) => { @@ -220,13 +203,12 @@ test.after.always(async (t) => { await destroyLocalTestDb({ ...t.context, }); - await cleanupTestIndex(t.context); }); // This test is a performance test designed to run with a large number of messages // in a memory constrained test environment, it is not intended to run as part of // the normal unit test suite. -test('writeGranulesFromMessage operates on 2k granules with 10 files each within 1GB of ram when an instance of EsClient is passed in and concurrency is set to 60 and db connections are set to 60', async (t) => { +test('writeGranulesFromMessage operates on 2k granules with 10 files each within 1GB of ram when concurrency is set to 60 and db connections are set to 60', async (t) => { const { cumulusMessages, knex, @@ -237,13 +219,11 @@ test('writeGranulesFromMessage operates on 2k granules with 10 files each within // Message must be completed or files will not update - const esClient = await getEsClient(); await pSettle(cumulusMessages.map((cumulusMessage) => () => writeGranulesFromMessage({ cumulusMessage, executionCumulusId, providerCumulusId, knex, - esClient, testOverrides: { stepFunctionUtils }, })), { concurrency: t.context.concurrency }); diff --git a/packages/api/webpack.config.js b/packages/api/webpack.config.js index b33246ed82a..ee2f8d27ad7 100644 --- a/packages/api/webpack.config.js +++ b/packages/api/webpack.config.js @@ -24,12 +24,10 @@ module.exports = { mode: process.env.PRODUCTION ? 'production' : 'development', entry: { app: './app/index.js', - bootstrap: './lambdas/bootstrap.js', bulkOperation: './lambdas/bulk-operation.js', cleanExecutions: './lambdas/cleanExecutions.js', createReconciliationReport: './lambdas/create-reconciliation-report.js', distribution: './app/distribution.js', - indexFromDatabase: './lambdas/index-from-database.js', manualConsumer: './lambdas/manual-consumer.js', messageConsumer: './lambdas/message-consumer.js', payloadLogger: './lambdas/payload-logger.js', diff --git a/packages/async-operations/.nycrc.json b/packages/async-operations/.nycrc.json index 6f46a3ffad1..3729b12e3da 100644 --- a/packages/async-operations/.nycrc.json +++ b/packages/async-operations/.nycrc.json @@ -2,6 +2,6 @@ "extends": "../../nyc.config.js", "statements": 97.0, "functions": 97.0, - "branches": 88.85, + "branches": 88.4, "lines": 97.0 } \ No newline at end of file diff --git a/packages/async-operations/src/async_operations.ts b/packages/async-operations/src/async_operations.ts index 0354f4a3b46..e0bbaf652dd 100644 --- a/packages/async-operations/src/async_operations.ts +++ b/packages/async-operations/src/async_operations.ts @@ -1,5 +1,4 @@ import { RunTaskCommandOutput } from '@aws-sdk/client-ecs'; -import { Knex } from 'knex'; import { FunctionConfiguration, GetFunctionConfigurationCommand } from '@aws-sdk/client-lambda'; import { ecs, s3, lambda } from '@cumulus/aws-client/services'; @@ -8,7 +7,6 @@ import { translateApiAsyncOperationToPostgresAsyncOperation, translatePostgresAsyncOperationToApiAsyncOperation, AsyncOperationPgModel, - createRejectableTransaction, } from '@cumulus/db'; import Logger from '@cumulus/logger'; import { ApiAsyncOperation, AsyncOperationType } from '@cumulus/types/api/async_operations'; @@ -19,13 +17,6 @@ import type { } from './types'; const { EcsStartTaskError, MissingRequiredArgument } = require('@cumulus/errors'); -const { - indexAsyncOperation, -} = require('@cumulus/es-client/indexer'); -const { - getEsClient, EsClient, -} = require('@cumulus/es-client/search'); - const logger = new Logger({ sender: '@cumulus/async-operation' }); type StartEcsTaskReturnType = Promise; @@ -129,7 +120,6 @@ export const createAsyncOperation = async ( stackName: string, systemBucket: string, knexConfig?: NodeJS.ProcessEnv, - esClient?: typeof EsClient, asyncOperationPgModel?: AsyncOperationPgModelObject } ): Promise> => { @@ -138,7 +128,6 @@ export const createAsyncOperation = async ( stackName, systemBucket, knexConfig = process.env, - esClient = await getEsClient(), asyncOperationPgModel = new AsyncOperationPgModel(), } = params; @@ -146,14 +135,9 @@ export const createAsyncOperation = async ( if (!systemBucket) throw new TypeError('systemBucket is required'); const knex = await getKnexClient({ env: knexConfig }); - return await createRejectableTransaction(knex, async (trx: Knex.Transaction) => { - const pgCreateObject = translateApiAsyncOperationToPostgresAsyncOperation(createObject); - const pgRecord = await asyncOperationPgModel.create(trx, pgCreateObject, ['*']); - const apiRecord = translatePostgresAsyncOperationToApiAsyncOperation(pgRecord[0]); - await indexAsyncOperation(esClient, apiRecord, process.env.ES_INDEX); - - return apiRecord; - }); + const pgCreateObject = translateApiAsyncOperationToPostgresAsyncOperation(createObject); + const pgRecord = await asyncOperationPgModel.create(knex, pgCreateObject, ['*']); + return translatePostgresAsyncOperationToApiAsyncOperation(pgRecord[0]); }; /** diff --git a/packages/async-operations/tests/test-async_operations.js b/packages/async-operations/tests/test-async_operations.js index 5411b4da1f7..a04caac38c5 100644 --- a/packages/async-operations/tests/test-async_operations.js +++ b/packages/async-operations/tests/test-async_operations.js @@ -23,11 +23,6 @@ const { migrationDir, } = require('@cumulus/db'); const { EcsStartTaskError, MissingRequiredArgument } = require('@cumulus/errors'); -const { Search } = require('@cumulus/es-client/search'); -const { - createTestIndex, - cleanupTestIndex, -} = require('@cumulus/es-client/testUtils'); const { getLambdaConfiguration, getLambdaEnvironmentVariables, @@ -55,15 +50,6 @@ test.before(async (t) => { systemBucket = randomString(); await s3().createBucket({ Bucket: systemBucket }); - const { esIndex, esClient } = await createTestIndex(); - t.context.esIndex = esIndex; - t.context.esClient = esClient; - t.context.esAsyncOperationsClient = new Search( - {}, - 'asyncOperation', - t.context.esIndex - ); - // Set up the mock ECS client ecsClient = ecs(); ecsClient.runTask = (params) => { @@ -75,7 +61,7 @@ test.before(async (t) => { t.context.functionConfig = { Environment: { Variables: { - ES_HOST: 'es-host', + Timeout: 300, }, }, }; @@ -94,7 +80,7 @@ test.beforeEach((t) => { status: 'RUNNING', taskArn: cryptoRandomString({ length: 5 }), description: 'testing', - operationType: 'ES Index', + operationType: 'Reconciliation Report', createdAt: Date.now(), updatedAt: Date.now(), }; @@ -103,7 +89,6 @@ test.beforeEach((t) => { test.after.always(async (t) => { sinon.restore(); await recursivelyDeleteS3Bucket(systemBucket); - await cleanupTestIndex(t.context); await destroyLocalTestDb({ knex: t.context.testKnex, knexAdmin: t.context.testKnexAdmin, @@ -125,7 +110,7 @@ test.serial('startAsyncOperation uploads the payload to S3', async (t) => { callerLambdaName: randomString(), lambdaName: randomString(), description: randomString(), - operationType: 'ES Index', + operationType: 'Reconciliation Report', payload, stackName, knexConfig: knexConfig, @@ -157,7 +142,7 @@ test.serial('The AsyncOperation start method starts an ECS task with the correct lambdaName, callerLambdaName, description: randomString(), - operationType: 'ES Index', + operationType: 'Reconciliation Report', payload, stackName, knexConfig: knexConfig, @@ -203,7 +188,7 @@ test.serial('The AsyncOperation start method starts an ECS task with the asyncOp lambdaName, callerLambdaName, description: randomString(), - operationType: 'ES Index', + operationType: 'Reconciliation Report', payload, stackName, knexConfig: knexConfig, @@ -240,7 +225,7 @@ test.serial('The startAsyncOperation method throws error and calls createAsyncOp callerLambdaName: randomString(), lambdaName: randomString(), description: randomString(), - operationType: 'ES Index', + operationType: 'Reconciliation Report', payload: {}, stackName: randomString(), knexConfig: knexConfig, @@ -275,10 +260,10 @@ test.serial('The startAsyncOperation method throws error and calls createAsyncOp ); }); -test('The startAsyncOperation writes records to all data stores', async (t) => { +test('The startAsyncOperation writes records to the database', async (t) => { const description = randomString(); const stackName = randomString(); - const operationType = 'ES Index'; + const operationType = 'Reconciliation Report'; const taskArn = randomString(); stubbedEcsRunTaskResult = { @@ -306,7 +291,7 @@ test('The startAsyncOperation writes records to all data stores', async (t) => { const expected = { description, id, - operationType: 'ES Index', + operationType: 'Reconciliation Report', status: 'RUNNING', taskArn, }; @@ -315,51 +300,6 @@ test('The startAsyncOperation writes records to all data stores', async (t) => { omit(asyncOperationPgRecord, omitList), translateApiAsyncOperationToPostgresAsyncOperation(omit(expected, omitList)) ); - const esRecord = await t.context.esAsyncOperationsClient.get(id); - t.deepEqual( - await t.context.esAsyncOperationsClient.get(id), - { - ...expected, - _id: esRecord._id, - timestamp: esRecord.timestamp, - updatedAt: esRecord.updatedAt, - createdAt: esRecord.createdAt, - } - ); -}); - -test.serial('The startAsyncOperation writes records with correct timestamps', async (t) => { - const description = randomString(); - const stackName = randomString(); - const operationType = 'ES Index'; - const taskArn = randomString(); - - stubbedEcsRunTaskResult = { - tasks: [{ taskArn }], - failures: [], - }; - - const { id } = await startAsyncOperation({ - asyncOperationTaskDefinition: randomString(), - cluster: randomString(), - callerLambdaName: randomString(), - lambdaName: randomString(), - description, - operationType, - payload: {}, - stackName, - knexConfig: knexConfig, - systemBucket, - }); - - const asyncOperationPgRecord = await t.context.asyncOperationPgModel.get( - t.context.testKnex, - { id } - ); - - const esRecord = await t.context.esAsyncOperationsClient.get(id); - t.is(asyncOperationPgRecord.created_at.getTime(), esRecord.createdAt); - t.is(asyncOperationPgRecord.updated_at.getTime(), esRecord.updatedAt); }); test.serial('The startAsyncOperation method returns the newly-generated record', async (t) => { @@ -377,7 +317,7 @@ test.serial('The startAsyncOperation method returns the newly-generated record', callerLambdaName: randomString(), lambdaName: randomString(), description: randomString(), - operationType: 'ES Index', + operationType: 'Reconciliation Report', payload: {}, stackName, knexConfig: knexConfig, @@ -400,7 +340,7 @@ test.serial('The startAsyncOperation method throws error if callerLambdaName par cluster: randomString, lambdaName: randomString, description: randomString(), - operationType: 'ES Index', + operationType: 'Reconciliation Report', payload: { x: randomString() }, stackName: randomString, knexConfig: knexConfig, @@ -420,7 +360,7 @@ test('getLambdaEnvironmentVariables returns expected environment variables', (t) const vars = getLambdaEnvironmentVariables(t.context.functionConfig); t.deepEqual(new Set(vars), new Set([ - { name: 'ES_HOST', value: 'es-host' }, + { name: 'Timeout', value: 300 }, ])); }); @@ -438,7 +378,7 @@ test.serial('ECS task params contain lambda environment variables when useLambda callerLambdaName: randomString(), lambdaName: randomString(), description: randomString(), - operationType: 'ES Index', + operationType: 'Reconciliation Report', payload: {}, useLambdaEnvironmentVariables: true, stackName, @@ -451,7 +391,7 @@ test.serial('ECS task params contain lambda environment variables when useLambda environmentOverrides[env.name] = env.value; }); - t.is(environmentOverrides.ES_HOST, 'es-host'); + t.is(environmentOverrides.Timeout, 300); }); test.serial('createAsyncOperation throws if stackName is not provided', async (t) => { @@ -495,63 +435,3 @@ test('createAsyncOperation throws if systemBucket is not provided', async (t) => { name: 'TypeError' } ); }); - -test.serial('createAsyncOperation() does not write to Elasticsearch if writing to PostgreSQL fails', async (t) => { - const { id, createObject } = t.context; - - const fakeAsyncOpPgModel = { - create: () => { - throw new Error('something bad'); - }, - }; - - const createParams = { - knex: t.context.testKnex, - asyncOperationPgModel: fakeAsyncOpPgModel, - createObject, - stackName: 'FakeStack', - systemBucket: 'FakeBucket', - }; - await t.throwsAsync( - createAsyncOperation(createParams), - { message: 'something bad' } - ); - - const dbRecords = await t.context.asyncOperationPgModel - .search(t.context.testKnex, { id }); - t.is(dbRecords.length, 0); - t.false(await t.context.esAsyncOperationsClient.exists( - id - )); -}); - -test.serial('createAsyncOperation() does not write to PostgreSQL if writing to Elasticsearch fails', async (t) => { - const { id, createObject } = t.context; - const fakeEsClient = { - initializeEsClient: () => Promise.resolve(), - client: { - index: () => { - throw new Error('ES something bad'); - }, - }, - }; - - const createParams = { - knex: t.context.testKnex, - createObject, - esClient: fakeEsClient, - stackName: 'FakeStack', - systemBucket: 'FakeBucket', - }; - await t.throwsAsync( - createAsyncOperation(createParams), - { message: 'ES something bad' } - ); - - const dbRecords = await t.context.asyncOperationPgModel - .search(t.context.testKnex, { id }); - t.is(dbRecords.length, 0); - t.false(await t.context.esAsyncOperationsClient.exists( - id - )); -}); diff --git a/packages/aws-client/src/services.ts b/packages/aws-client/src/services.ts index ff6bfd5572c..04f97ee8ea2 100644 --- a/packages/aws-client/src/services.ts +++ b/packages/aws-client/src/services.ts @@ -15,7 +15,6 @@ import { SNS } from '@aws-sdk/client-sns'; import { STS } from '@aws-sdk/client-sts'; import { ECS } from '@aws-sdk/client-ecs'; import { EC2 } from '@aws-sdk/client-ec2'; -import { ElasticsearchService } from '@aws-sdk/client-elasticsearch-service'; import awsClient from './client'; @@ -31,7 +30,6 @@ export const dynamodbDocClient = (docClientOptions?: TranslateConfig, dynamoOpti docClientOptions ); export const cf = awsClient(CloudFormation, '2010-05-15'); -export const es = awsClient(ElasticsearchService, '2015-01-01'); export const kinesis = awsClient(Kinesis, '2013-12-02'); export const kms = awsClient(KMS, '2014-11-01'); export const lambda = awsClient(Lambda, '2015-03-31'); diff --git a/packages/aws-client/tests/test-services.js b/packages/aws-client/tests/test-services.js index 994d27a68a0..9677f99ed58 100644 --- a/packages/aws-client/tests/test-services.js +++ b/packages/aws-client/tests/test-services.js @@ -6,7 +6,6 @@ const { CloudFormation } = require('@aws-sdk/client-cloudformation'); const { DynamoDB } = require('@aws-sdk/client-dynamodb'); const { ECS } = require('@aws-sdk/client-ecs'); const { EC2 } = require('@aws-sdk/client-ec2'); -const { ElasticsearchService } = require('@aws-sdk/client-elasticsearch-service'); const { Kinesis } = require('@aws-sdk/client-kinesis'); const { Lambda } = require('@aws-sdk/client-lambda'); const { S3 } = require('@aws-sdk/client-s3'); @@ -188,27 +187,6 @@ test('ec2() service defaults to localstack in test mode', async (t) => { ); }); -test('es() service defaults to localstack in test mode', async (t) => { - const es = services.es(); - const { - credentials, - endpoint, - } = localStackAwsClientOptions(ElasticsearchService); - t.like( - await es.config.credentials(), - credentials - ); - const esEndpoint = await es.config.endpoint(); - const localSatckEndpoint = new URL(endpoint); - t.like( - esEndpoint, - { - hostname: localSatckEndpoint.hostname, - port: Number.parseInt(localSatckEndpoint.port, 10), - } - ); -}); - test('kinesis() service defaults to localstack in test mode', async (t) => { const kinesis = services.kinesis(); const { diff --git a/packages/cmr-client/src/CMR.ts b/packages/cmr-client/src/CMR.ts index b9a04b1bb4a..b8096f6efdd 100644 --- a/packages/cmr-client/src/CMR.ts +++ b/packages/cmr-client/src/CMR.ts @@ -41,7 +41,7 @@ export interface CMRConstructorParams { passwordSecretName?: string provider: string, token?: string, - username: string, + username?: string, oauthProvider: string, } @@ -67,11 +67,13 @@ export interface CMRConstructorParams { * clientId: 'my-clientId', * token: 'cmr_or_launchpad_token' * }); + * TODO: this should be subclassed or refactored to a functional style + * due to branch logic/complexity in token vs password/username handling */ export class CMR { clientId: string; provider: string; - username: string; + username?: string; oauthProvider: string; password?: string; passwordSecretName?: string; @@ -79,17 +81,6 @@ export class CMR { /** * The constructor for the CMR class - * - * @param {Object} params - * @param {string} params.provider - the CMR provider id - * @param {string} params.clientId - the CMR clientId - * @param {string} params.username - CMR username, not used if token is provided - * @param {string} params.passwordSecretName - CMR password secret, not used if token is provided - * @param {string} params.password - CMR password, not used if token or - * passwordSecretName is provided - * @param {string} params.token - CMR or Launchpad token, - * if not provided, CMR username and password are used to get a cmr token - * @param {string} params.oauthProvider - Oauth provider: earthdata or launchpad */ constructor(params: CMRConstructorParams) { this.clientId = params.clientId; @@ -131,6 +122,12 @@ export class CMR { * @returns {Promise.} the token */ async getToken(): Promise { + if (this.oauthProvider === 'launchpad') { + return this.token; + } + if (!this.username) { + throw new Error('Username not specified for non-launchpad CMR client'); + } return this.token ? this.token : updateToken(this.username, await this.getCmrPassword()); diff --git a/packages/cmr-client/src/CMRSearchConceptQueue.ts b/packages/cmr-client/src/CMRSearchConceptQueue.ts index ea0b142bd19..543b967ad6a 100644 --- a/packages/cmr-client/src/CMRSearchConceptQueue.ts +++ b/packages/cmr-client/src/CMRSearchConceptQueue.ts @@ -2,18 +2,12 @@ import { CMR, CMRConstructorParams } from './CMR'; /** * Shim to correctly add a default provider_short_name to the input searchParams - * - * @param {Object} params - * @param {URLSearchParams} params.searchParams - input search - * parameters for searchConceptQueue. This parameter can be either a - * URLSearchParam object or a plain Object. - * @returns {URLSearchParams} - input object appeneded with a default provider_short_name */ export const providerParams = ({ searchParams = new URLSearchParams(), cmrSettings, }: { - searchParams: URLSearchParams, + searchParams?: URLSearchParams, cmrSettings: { provider: string } @@ -28,7 +22,7 @@ export const providerParams = ({ export interface CMRSearchConceptQueueConstructorParams { cmrSettings: CMRConstructorParams, type: string, - searchParams: URLSearchParams, + searchParams?: URLSearchParams, format?: string } @@ -49,18 +43,18 @@ export interface CMRSearchConceptQueueConstructorParams { * format: 'json' * }); */ -export class CMRSearchConceptQueue { +export class CMRSearchConceptQueue { type: string; params: URLSearchParams; format?: string; - items: unknown[]; + items: (T | null)[]; CMR: CMR; /** * The constructor for the CMRSearchConceptQueue class * * @param {Object} params - * @param {string} params.cmrSettings - the CMR settings for the requests - the provider, + * @param {Object} params.cmrSettings - the CMR settings for the requests - the provider, * clientId, and either launchpad token or EDL username and password * @param {string} params.type - the type of search 'granule' or 'collection' * @param {URLSearchParams} [params.searchParams={}] - the search parameters @@ -84,10 +78,12 @@ export class CMRSearchConceptQueue { * This does not remove the object from the queue. When there are no more * items in the queue, returns 'null'. * - * @returns {Promise} an item from the CMR search */ - async peek(): Promise { + async peek(): Promise { if (this.items.length === 0) await this.fetchItems(); + if (this.items[0] === null) { + return null; + } return this.items[0]; } @@ -95,12 +91,15 @@ export class CMRSearchConceptQueue { * Remove the next item from the queue * * When there are no more items in the queue, returns `null`. - * - * @returns {Promise} an item from the CMR search */ - async shift(): Promise { + async shift(): Promise { if (this.items.length === 0) await this.fetchItems(); - return this.items.shift(); + const item = this.items.shift(); + // eslint-disable-next-line lodash/prefer-is-nil + if (item === null || item === undefined) { + return null; + } + return item; } /** @@ -116,7 +115,7 @@ export class CMRSearchConceptQueue { this.format, false ); - this.items = results; + this.items = results as T[]; const paramsPageNum = this.params.get('page_num') ?? '0'; this.params.set('page_num', String(Number(paramsPageNum) + 1)); diff --git a/packages/cmr-client/tests/test-CMR.js b/packages/cmr-client/tests/test-CMR.js index ef1641c657b..1f787c1e3af 100644 --- a/packages/cmr-client/tests/test-CMR.js +++ b/packages/cmr-client/tests/test-CMR.js @@ -166,7 +166,7 @@ test('getReadHeaders returns clientId and token for launchpad', (t) => { }); test.serial('ingestUMMGranule() returns CMRInternalError when CMR is down', async (t) => { - const cmrSearch = new CMR({ provider: 'my-provider', token: 'abc', clientId: 'client' }); + const cmrSearch = new CMR({ oauthProvider: 'launchpad', token: 'abc', clientId: 'client' }); const ummgMetadata = { GranuleUR: 'asdf' }; @@ -192,7 +192,7 @@ test.serial('ingestUMMGranule() returns CMRInternalError when CMR is down', asyn }); test.serial('ingestUMMGranule() throws an exception if the input fails validation', async (t) => { - const cmrSearch = new CMR({ provider: 'my-provider', token: 'abc', clientId: 'client' }); + const cmrSearch = new CMR({ oauthProvider: 'launchpad', token: 'abc', clientId: 'client' }); const ummgMetadata = { GranuleUR: 'asdf' }; @@ -257,3 +257,17 @@ test('getToken returns a token when the user\'s token is provided', async (t) => t.is(await cmrObj.getToken(), 'abcde'); }); + +test('getToken throws if no username is provided when using Earthdata Login', async (t) => { + const cmrObj = new CMR({ + provider: 'CUMULUS', + clientId: 'clientId', + password: 'password', + oauthProvider: 'earthdata', + }); + + await t.throwsAsync( + () => cmrObj.getToken(), + { message: 'Username not specified for non-launchpad CMR client' } + ); +}); diff --git a/packages/cmrjs/src/cmr-utils.js b/packages/cmrjs/src/cmr-utils.js index ebc2ef2ccb4..be6a1ed4031 100644 --- a/packages/cmrjs/src/cmr-utils.js +++ b/packages/cmrjs/src/cmr-utils.js @@ -469,6 +469,14 @@ function generateFileUrl({ return undefined; } +/** + * @typedef {Object} OnlineAccessUrl + * @property {string} URL - The generated file URL. + * @property {string} URLDescription - The description of the URL (used by ECHO10). + * @property {string} Description - The description of the URL (used by UMMG). + * @property {string} Type - The type of the URL (used by ECHO10/UMMG). + */ + /** * Construct online access url for a given file and a url type. * @@ -479,8 +487,8 @@ function generateFileUrl({ * @param {Object} params.urlType - url type, distribution or s3 * @param {distributionBucketMap} params.distributionBucketMap - Object with bucket:tea-path mapping * for all distribution bucketss - * @param {boolean} params.useDirectS3Type - indicate if direct s3 access type is used - * @returns {(Object | undefined)} online access url object, undefined if no URL exists + * @param {boolean} [params.useDirectS3Type] - indicate if direct s3 access type is used + * @returns {(OnlineAccessUrl | undefined)} online access url object, undefined if no URL exists */ function constructOnlineAccessUrl({ file, @@ -762,15 +770,17 @@ async function updateUMMGMetadata({ * Helper to build an CMR settings object, used to initialize CMR. * * @param {Object} cmrConfig - CMR configuration object - * @param {string} cmrConfig.oauthProvider - Oauth provider: launchpad or earthdata - * @param {string} cmrConfig.provider - the CMR provider - * @param {string} cmrConfig.clientId - Client id for CMR requests - * @param {string} cmrConfig.passphraseSecretName - Launchpad passphrase secret name - * @param {string} cmrConfig.api - Launchpad api - * @param {string} cmrConfig.certificate - Launchpad certificate - * @param {string} cmrConfig.username - EDL username - * @param {string} cmrConfig.passwordSecretName - CMR password secret name - * @returns {Promise} object to create CMR instance - contains the + * @param {string} [cmrConfig.oauthProvider] - Oauth provider: launchpad or earthdata + * @param {string} [cmrConfig.provider] - the CMR provider + * @param {string} [cmrConfig.clientId] - Client id for CMR requests + * @param {string} [cmrConfig.passphraseSecretName] - Launchpad passphrase secret name + * @param {string} [cmrConfig.api] - Launchpad api + * @param {string} [cmrConfig.certificate] - Launchpad certificate + * @param {string} [cmrConfig.username] - EDL username + * @param {string} [cmrConfig.passwordSecretName] - CMR password secret name + * @returns {Promise} + * object to + * create CMR instance - contains the * provider, clientId, and either launchpad token or EDL username and * password */ diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index df405f0f51f..417312ba625 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -12,6 +12,7 @@ export { fakeGranuleRecordFactory, fakePdrRecordFactory, fakeProviderRecordFactory, + fakeReconciliationReportRecordFactory, fakeRuleRecordFactory, generateLocalTestDb, } from './test-utils'; @@ -31,7 +32,6 @@ export { export { BaseRecord, } from './types/base'; - export { PostgresAsyncOperation, PostgresAsyncOperationRecord, @@ -67,6 +67,11 @@ export { export { PostgresGranuleExecution, } from './types/granule-execution'; +export { + PostgresReconciliationReport, + PostgresReconciliationReportRecord, +} from './types/reconciliation_report'; + export { translateApiAsyncOperationToPostgresAsyncOperation, translatePostgresAsyncOperationToApiAsyncOperation, @@ -75,12 +80,10 @@ export { translateApiFiletoPostgresFile, translatePostgresFileToApiFile, } from './translate/file'; - export { translateApiCollectionToPostgresCollection, translatePostgresCollectionToApiCollection, } from './translate/collections'; - export { translateApiProviderToPostgresProvider, translatePostgresProviderToApiProvider, @@ -105,9 +108,14 @@ export { translateApiPdrToPostgresPdr, translatePostgresPdrToApiPdr, } from './translate/pdr'; +export { + translateApiReconReportToPostgresReconReport, + translatePostgresReconReportToApiReconReport, +} from './translate/reconciliation_reports'; export { getCollectionsByGranuleIds, + getUniqueCollectionsByGranuleFilter, } from './lib/collection'; export { @@ -141,20 +149,32 @@ export { QuerySearchClient, } from './lib/QuerySearchClient'; export { - BaseSearch, -} from './search/BaseSearch'; + AsyncOperationSearch, +} from './search/AsyncOperationSearch'; +export { + CollectionSearch, +} from './search/CollectionSearch'; export { ExecutionSearch, } from './search/ExecutionSearch'; export { GranuleSearch, } from './search/GranuleSearch'; +export { + PdrSearch, +} from './search/PdrSearch'; +export { + ProviderSearch, +} from './search/ProviderSearch'; +export { + RuleSearch, +} from './search/RuleSearch'; export { StatsSearch, } from './search/StatsSearch'; export { - CollectionSearch, -} from './search/CollectionSearch'; + ReconciliationReportSearch, +} from './search/ReconciliationReportSearch'; export { AsyncOperationPgModel } from './models/async_operation'; export { BasePgModel } from './models/base'; @@ -165,4 +185,5 @@ export { GranulePgModel } from './models/granule'; export { GranulesExecutionsPgModel } from './models/granules-executions'; export { PdrPgModel } from './models/pdr'; export { ProviderPgModel } from './models/provider'; +export { ReconciliationReportPgModel } from './models/reconciliation_report'; export { RulePgModel } from './models/rule'; diff --git a/packages/db/src/lib/QuerySearchClient.ts b/packages/db/src/lib/QuerySearchClient.ts index acb890e144d..470c7b30e9c 100644 --- a/packages/db/src/lib/QuerySearchClient.ts +++ b/packages/db/src/lib/QuerySearchClient.ts @@ -43,7 +43,6 @@ class QuerySearchClient { * * This does not remove the object from the queue. * - * @returns {Promise} - record from PostgreSQL table */ async peek() { if (this.records.length === 0) await this.fetchRecords(); @@ -53,7 +52,6 @@ class QuerySearchClient { /** * Remove and return the next item in the results * - * @returns {Promise} - record from PostgreSQL table */ async shift() { if (this.records.length === 0) await this.fetchRecords(); diff --git a/packages/db/src/lib/collection.ts b/packages/db/src/lib/collection.ts index f8b1db297f8..d43c1ab269d 100644 --- a/packages/db/src/lib/collection.ts +++ b/packages/db/src/lib/collection.ts @@ -1,6 +1,8 @@ import { Knex } from 'knex'; import Logger from '@cumulus/logger'; +import { deconstructCollectionId } from '@cumulus/message/Collections'; + import { RetryOnDbConnectionTerminateError } from './retry'; import { TableNames } from '../tables'; @@ -27,3 +29,56 @@ export const getCollectionsByGranuleIds = async ( .groupBy(`${collectionsTable}.cumulus_id`); return await RetryOnDbConnectionTerminateError(query, {}, log); }; + +// TODO - This function is going to be super-non-performant +// We need to identify the specific need here and see if we can optimize + +export const getUniqueCollectionsByGranuleFilter = async (params: { + startTimestamp?: string, + endTimestamp?: string, + collectionIds?: string[], + granuleIds?: string[], + providers?: string[], + knex: Knex, +}) => { + const { knex } = params; + const collectionsTable = TableNames.collections; + const granulesTable = TableNames.granules; + const providersTable = TableNames.providers; + + const query = knex(collectionsTable) + .distinct(`${collectionsTable}.*`) + .innerJoin(granulesTable, `${collectionsTable}.cumulus_id`, `${granulesTable}.collection_cumulus_id`); + + if (params.startTimestamp) { + query.where(`${granulesTable}.updated_at`, '>=', params.startTimestamp); + } + if (params.endTimestamp) { + query.where(`${granulesTable}.updated_at`, '<=', params.endTimestamp); + } + + // Filter by collectionIds + if (params.collectionIds && params.collectionIds.length > 0) { + const collectionNameVersionPairs = params.collectionIds.map((id) => + deconstructCollectionId(id)); + + query.whereIn( + [`${collectionsTable}.name`, `${collectionsTable}.version`], + collectionNameVersionPairs.map(({ name, version }) => [name, version]) + ); + } + + // Filter by granuleIds + if (params.granuleIds && params.granuleIds.length > 0) { + query.whereIn(`${granulesTable}.granule_id`, params.granuleIds); + } + + // Filter by provider names + if (params.providers && params.providers.length > 0) { + query.innerJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`); + query.whereIn(`${providersTable}.name`, params.providers); + } + + query.orderBy([`${collectionsTable}.name`, `${collectionsTable}.version`]); + return query; +}; diff --git a/packages/db/src/lib/execution.ts b/packages/db/src/lib/execution.ts index dd824e52966..7970a4fb881 100644 --- a/packages/db/src/lib/execution.ts +++ b/packages/db/src/lib/execution.ts @@ -17,42 +17,30 @@ export interface ArnRecord { const log = new Logger({ sender: '@cumulus/db/lib/execution' }); /** - * Returns execution info sorted by most recent first for an input - * Granule Cumulus ID. - * - * @param {Object} params - * @param {Knex | Knex.Transaction} params.knexOrTransaction - * Knex client for reading from RDS database - * @param {Array} params.executionColumns - Columns to return from executions table - * @param {number} params.granuleCumulusId - The primary ID for a Granule - * @param {number} [params.limit] - limit to number of executions to query - * @returns {Promise[]>} - * Array of arn objects with the most recent first. + * Returns execution records sorted by most recent first for an input + * set of Granule Cumulus IDs. + * @returns Array of arn objects with the most recent first. */ -export const getExecutionInfoByGranuleCumulusId = async ({ +export const getExecutionInfoByGranuleCumulusIds = async ({ knexOrTransaction, - granuleCumulusId, - executionColumns = ['arn'], + granuleCumulusIds, limit, }: { knexOrTransaction: Knex | Knex.Transaction, - granuleCumulusId: number, - executionColumns: string[], + granuleCumulusIds: number[], limit?: number -}): Promise[]> => { +}): Promise<{ granule_cumulus_id: number, url: string }[]> => { const knexQuery = knexOrTransaction(TableNames.executions) - .column(executionColumns.map((column) => `${TableNames.executions}.${column}`)) - .where(`${TableNames.granules}.cumulus_id`, granuleCumulusId) + .column([ + `${TableNames.executions}.url`, + `${TableNames.granulesExecutions}.granule_cumulus_id`, + ]) + .whereIn(`${TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusIds) .join( TableNames.granulesExecutions, `${TableNames.executions}.cumulus_id`, `${TableNames.granulesExecutions}.execution_cumulus_id` ) - .join( - TableNames.granules, - `${TableNames.granules}.cumulus_id`, - `${TableNames.granulesExecutions}.granule_cumulus_id` - ) .orderBy(`${TableNames.executions}.timestamp`, 'desc'); if (limit) { knexQuery.limit(limit); @@ -61,33 +49,37 @@ export const getExecutionInfoByGranuleCumulusId = async ({ }; /** - * Returns a list of executionArns sorted by most recent first, for an input + * Returns execution records sorted by most recent first for an input * Granule Cumulus ID. * - * @param {Knex | Knex.Transaction} knexOrTransaction + * @param {Object} params + * @param {Knex | Knex.Transaction} params.knexOrTransaction * Knex client for reading from RDS database - * @param {number} granuleCumulusId - The primary ID for a Granule - * @param {number} limit - limit to number of executions to query - * @returns {Promise} - Array of arn objects with the most recent first. + * @param {Array} params.executionColumns - Columns to return from executions table + * @param {number} params.granuleCumulusId - The primary ID for a Granule + * @param {number} [params.limit] - limit to number of executions to query + * @returns {Promise[]>} + * Array of arn objects with the most recent first. */ -export const getExecutionArnsByGranuleCumulusId = async ( +export const getExecutionInfoByGranuleCumulusId = async ({ + knexOrTransaction, + granuleCumulusId, + executionColumns = ['arn'], + limit, +}: { knexOrTransaction: Knex | Knex.Transaction, - granuleCumulusId: Number, + granuleCumulusId: number, + executionColumns: string[], limit?: number -): Promise => { +}): Promise[]> => { const knexQuery = knexOrTransaction(TableNames.executions) - .select(`${TableNames.executions}.arn`) - .where(`${TableNames.granules}.cumulus_id`, granuleCumulusId) + .column(executionColumns.map((column) => `${TableNames.executions}.${column}`)) + .where(`${TableNames.granulesExecutions}.granule_cumulus_id`, granuleCumulusId) .join( TableNames.granulesExecutions, `${TableNames.executions}.cumulus_id`, `${TableNames.granulesExecutions}.execution_cumulus_id` ) - .join( - TableNames.granules, - `${TableNames.granules}.cumulus_id`, - `${TableNames.granulesExecutions}.granule_cumulus_id` - ) .orderBy(`${TableNames.executions}.timestamp`, 'desc'); if (limit) { knexQuery.limit(limit); diff --git a/packages/db/src/lib/granule.ts b/packages/db/src/lib/granule.ts index e0a77df6cde..22015e53f07 100644 --- a/packages/db/src/lib/granule.ts +++ b/packages/db/src/lib/granule.ts @@ -206,39 +206,31 @@ export const getApiGranuleExecutionCumulusIds = async ( /** * Helper to build a query to search granules by various API granule record properties. - * - * @param {Knex} knex - DB client - * @param {Object} searchParams - * @param {string | Array} [searchParams.collectionIds] - Collection ID - * @param {string | Array} [searchParams.granuleIds] - array of granule IDs - * @param {string} [searchParams.providerNames] - Provider names - * @param {UpdatedAtRange} [searchParams.updatedAtRange] - Date range for updated_at column - * @param {string} [searchParams.status] - Granule status to search by - * @param {string | Array} [sortByFields] - Field(s) to sort by - * @returns {Knex.QueryBuilder} */ -export const getGranulesByApiPropertiesQuery = ( +export const getGranulesByApiPropertiesQuery = ({ + knex, + searchParams, + sortByFields = [], + temporalBoundByCreatedAt = false, +} : { knex: Knex, - { - collectionIds, - granuleIds, - providerNames, - updatedAtRange = {}, - status, - }: { + searchParams: { + collate?: string, collectionIds?: string | string[], granuleIds?: string | string[], providerNames?: string[], + status?: string updatedAtRange?: UpdatedAtRange, - status?: string, }, - sortByFields?: string | string[] -): Knex.QueryBuilder => { + sortByFields?: string | string[], + temporalBoundByCreatedAt?: boolean, +}) : Knex.QueryBuilder => { const { granules: granulesTable, collections: collectionsTable, providers: providersTable, } = TableNames; + const temporalColumn = temporalBoundByCreatedAt ? 'created_at' : 'updated_at'; return knex(granulesTable) .select(`${granulesTable}.*`) .select({ @@ -249,8 +241,8 @@ export const getGranulesByApiPropertiesQuery = ( .innerJoin(collectionsTable, `${granulesTable}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) .leftJoin(providersTable, `${granulesTable}.provider_cumulus_id`, `${providersTable}.cumulus_id`) .modify((queryBuilder) => { - if (collectionIds) { - const collectionIdFilters = [collectionIds].flat(); + if (searchParams.collectionIds) { + const collectionIdFilters = [searchParams.collectionIds].flat(); const collectionIdConcatField = `(${collectionsTable}.name || '${collectionIdSeparator}' || ${collectionsTable}.version)`; const collectionIdInClause = collectionIdFilters.map(() => '?').join(','); queryBuilder.whereRaw( @@ -258,28 +250,34 @@ export const getGranulesByApiPropertiesQuery = ( collectionIdFilters ); } - if (granuleIds) { - const granuleIdFilters = [granuleIds].flat(); + if (searchParams.granuleIds) { + const granuleIdFilters = [searchParams.granuleIds].flat(); queryBuilder.where((nestedQueryBuilder) => { granuleIdFilters.forEach((granuleId) => { nestedQueryBuilder.orWhere(`${granulesTable}.granule_id`, 'LIKE', `%${granuleId}%`); }); }); } - if (providerNames) { - queryBuilder.whereIn(`${providersTable}.name`, providerNames); + if (searchParams.providerNames) { + queryBuilder.whereIn(`${providersTable}.name`, searchParams.providerNames); } - if (updatedAtRange.updatedAtFrom) { - queryBuilder.where(`${granulesTable}.updated_at`, '>=', updatedAtRange.updatedAtFrom); + if (searchParams?.updatedAtRange?.updatedAtFrom) { + queryBuilder.where(`${granulesTable}.${temporalColumn}`, '>=', searchParams.updatedAtRange.updatedAtFrom); } - if (updatedAtRange.updatedAtTo) { - queryBuilder.where(`${granulesTable}.updated_at`, '<=', updatedAtRange.updatedAtTo); + if (searchParams?.updatedAtRange?.updatedAtTo) { + queryBuilder.where(`${granulesTable}.${temporalColumn}`, '<=', searchParams.updatedAtRange.updatedAtTo); } - if (status) { - queryBuilder.where(`${granulesTable}.status`, status); + if (searchParams.status) { + queryBuilder.where(`${granulesTable}.status`, searchParams.status); } if (sortByFields) { - queryBuilder.orderBy([sortByFields].flat()); + if (!searchParams.collate) { + queryBuilder.orderBy([sortByFields].flat()); + } else { + [sortByFields].flat().forEach((field) => { + queryBuilder.orderByRaw(`${field} collate \"${searchParams.collate}\"`); + }); + } } }) .groupBy(`${granulesTable}.cumulus_id`) diff --git a/packages/db/src/migrations/20240814185217_create_reconciliation_reports_table.ts b/packages/db/src/migrations/20240814185217_create_reconciliation_reports_table.ts new file mode 100644 index 00000000000..ebf624c3f1f --- /dev/null +++ b/packages/db/src/migrations/20240814185217_create_reconciliation_reports_table.ts @@ -0,0 +1,38 @@ +import { Knex } from 'knex'; + +export const up = async (knex: Knex): Promise => { + await knex.schema.createTable('reconciliation_reports', (table) => { + table + .increments('cumulus_id') + .primary(); + table + .text('name') + .comment('Reconciliation Report name') + .notNullable(); + table + .enum('type', + ['Granule Inventory', 'Granule Not Found', 'Internal', 'Inventory', 'ORCA Backup']) + .comment('Type of Reconciliation Report') + .notNullable(); + table + .enum('status', ['Generated', 'Pending', 'Failed']) + .comment('Status of Reconciliation Report') + .notNullable(); + table + .text('location') + .comment('Location of Reconciliation Report'); + table + .jsonb('error') + .comment('Error object'); + // adds "created_at" and "updated_at" columns automatically + table + .timestamps(false, true); + table.index('status'); + table.index('updated_at'); + table.unique(['name']); + }); +}; + +export const down = async (knex: Knex): Promise => { + await knex.schema.dropTableIfExists('reconciliation_reports'); +}; diff --git a/packages/db/src/models/file.ts b/packages/db/src/models/file.ts index 472f9f4c68a..01de6c6d22e 100644 --- a/packages/db/src/models/file.ts +++ b/packages/db/src/models/file.ts @@ -22,6 +22,18 @@ class FilePgModel extends BasePgModel { .merge() .returning('*'); } + /** + * Retrieves all files for all granules given + */ + searchByGranuleCumulusIds( + knexOrTrx: Knex | Knex.Transaction, + granule_cumulus_ids: number[], + columns: string | string[] = '*' + ): Promise { + return knexOrTrx(this.tableName) + .select(columns) + .whereIn('granule_cumulus_id', granule_cumulus_ids); + } } export { FilePgModel }; diff --git a/packages/db/src/models/reconciliation_report.ts b/packages/db/src/models/reconciliation_report.ts new file mode 100644 index 00000000000..b9cf548f8ca --- /dev/null +++ b/packages/db/src/models/reconciliation_report.ts @@ -0,0 +1,37 @@ +import { Knex } from 'knex'; +import { BasePgModel } from './base'; +import { TableNames } from '../tables'; + +import { + PostgresReconciliationReport, + PostgresReconciliationReportRecord, +} from '../types/reconciliation_report'; + +// eslint-disable-next-line max-len +class ReconciliationReportPgModel extends BasePgModel { + constructor() { + super({ + tableName: TableNames.reconciliationReports, + }); + } + + create( + knexOrTransaction: Knex | Knex.Transaction, + item: PostgresReconciliationReport + ): Promise { + return super.create(knexOrTransaction, item, '*') as Promise; + } + + upsert( + knexOrTransaction: Knex | Knex.Transaction, + reconciliationReport: PostgresReconciliationReport + ): Promise { + return knexOrTransaction(this.tableName) + .insert(reconciliationReport) + .onConflict('name') + .merge() + .returning('*'); + } +} + +export { ReconciliationReportPgModel }; diff --git a/packages/db/src/search/AsyncOperationSearch.ts b/packages/db/src/search/AsyncOperationSearch.ts new file mode 100644 index 00000000000..17ded787f5a --- /dev/null +++ b/packages/db/src/search/AsyncOperationSearch.ts @@ -0,0 +1,62 @@ +import { Knex } from 'knex'; +import pick from 'lodash/pick'; + +import { ApiAsyncOperation } from '@cumulus/types/api/async_operations'; +import Logger from '@cumulus/logger'; + +import { BaseSearch } from './BaseSearch'; +import { DbQueryParameters, QueryEvent } from '../types/search'; +import { PostgresAsyncOperationRecord } from '../types/async_operation'; +import { translatePostgresAsyncOperationToApiAsyncOperation } from '../translate/async_operations'; + +const log = new Logger({ sender: '@cumulus/db/AsyncOperationSearch' }); + +/** + * Class to build and execute db search query for asyncOperation + */ +export class AsyncOperationSearch extends BaseSearch { + constructor(event: QueryEvent) { + super(event, 'asyncOperation'); + } + + /** + * Build queries for infix and prefix + * + * @param params + * @param params.countQuery - query builder for getting count + * @param params.searchQuery - query builder for search + * @param [params.dbQueryParameters] - db query parameters + */ + protected buildInfixPrefixQuery(params: { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + dbQueryParameters?: DbQueryParameters, + }) { + const { countQuery, searchQuery, dbQueryParameters } = params; + const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters; + if (infix) { + [countQuery, searchQuery].forEach((query) => query.whereRaw(`${this.tableName}.id::text like ?`, `%${infix}%`)); + } + if (prefix) { + [countQuery, searchQuery].forEach((query) => query.whereRaw(`${this.tableName}.id::text like ?`, `${prefix}%`)); + } + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres records returned from query + * @returns translated api records + */ + protected translatePostgresRecordsToApiRecords(pgRecords: PostgresAsyncOperationRecord[]) + : Partial[] { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + const { fields } = this.dbQueryParameters; + const apiRecords = pgRecords.map((item: PostgresAsyncOperationRecord) => { + const pgAsyncOperation = item; + const apiRecord = translatePostgresAsyncOperationToApiAsyncOperation(pgAsyncOperation); + return fields ? pick(apiRecord, fields) : apiRecord; + }); + return apiRecords; + } +} diff --git a/packages/db/src/search/BaseSearch.ts b/packages/db/src/search/BaseSearch.ts index 663c0ebfb03..5e6177454a0 100644 --- a/packages/db/src/search/BaseSearch.ts +++ b/packages/db/src/search/BaseSearch.ts @@ -28,12 +28,14 @@ export const typeToTable: { [key: string]: string } = { pdr: TableNames.pdrs, provider: TableNames.providers, rule: TableNames.rules, + reconciliationReport: TableNames.reconciliationReports, }; /** * Class to build and execute db search query */ -class BaseSearch { + +abstract class BaseSearch { readonly type: string; readonly tableName: string; readonly queryStringParameters: QueryStringParameters; @@ -64,6 +66,16 @@ class BaseSearch { || terms?.collectionVersion); } + /** + * check if joined executions table search is needed + * + * @returns whether execution search is needed + */ + protected searchExecution(): boolean { + const { not, term, terms } = this.dbQueryParameters; + return !!(not?.executionArn || term?.executionArn || terms?.executionArn); + } + /** * check if joined pdrs table search is needed * @@ -140,14 +152,18 @@ class BaseSearch { * Build basic query * * @param knex - DB client - * @throws - function is not implemented + * @returns queries for getting count and search result */ protected buildBasicQuery(knex: Knex): { countQuery?: Knex.QueryBuilder, searchQuery: Knex.QueryBuilder, } { - log.debug(`buildBasicQuery is not implemented ${knex.constructor.name}`); - throw new Error('buildBasicQuery is not implemented'); + const countQuery = knex(this.tableName) + .count('*'); + + const searchQuery = knex(this.tableName) + .select(`${this.tableName}.*`); + return { countQuery, searchQuery }; } /** @@ -191,6 +207,9 @@ class BaseSearch { case 'collectionVersion': [countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.collection_cumulus_id`)); break; + case 'executionArn': + [countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.execution_cumulus_id`)); + break; case 'providerName': [countQuery, searchQuery].forEach((query) => query?.[queryMethod](`${this.tableName}.provider_cumulus_id`)); break; @@ -233,13 +252,12 @@ class BaseSearch { const { range = {} } = dbQueryParameters ?? this.dbQueryParameters; Object.entries(range).forEach(([name, rangeValues]) => { - if (rangeValues.gte) { - countQuery?.where(`${this.tableName}.${name}`, '>=', rangeValues.gte); - searchQuery.where(`${this.tableName}.${name}`, '>=', rangeValues.gte); + const { gte, lte } = rangeValues; + if (gte) { + [countQuery, searchQuery].forEach((query) => query?.where(`${this.tableName}.${name}`, '>=', gte)); } - if (rangeValues.lte) { - countQuery?.where(`${this.tableName}.${name}`, '<=', rangeValues.lte); - searchQuery.where(`${this.tableName}.${name}`, '<=', rangeValues.lte); + if (lte) { + [countQuery, searchQuery].forEach((query) => query?.where(`${this.tableName}.${name}`, '<=', lte)); } }); } @@ -276,6 +294,9 @@ class BaseSearch { case 'collectionVersion': [countQuery, searchQuery].forEach((query) => query?.where(`${collectionsTable}.version`, value)); break; + case 'executionArn': + [countQuery, searchQuery].forEach((query) => query?.where(`${executionsTable}.arn`, value)); + break; case 'providerName': [countQuery, searchQuery].forEach((query) => query?.where(`${providersTable}.name`, value)); break; @@ -284,7 +305,7 @@ class BaseSearch { break; case 'error.Error': [countQuery, searchQuery] - .forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' = '${value}'`)); + .forEach((query) => value && query?.whereRaw(`${this.tableName}.error->>'Error' = ?`, value)); break; case 'asyncOperationId': [countQuery, searchQuery].forEach((query) => query?.where(`${asyncOperationsTable}.id`, value)); @@ -339,6 +360,9 @@ class BaseSearch { Object.entries(omit(terms, ['collectionName', 'collectionVersion'])).forEach(([name, value]) => { switch (name) { + case 'executionArn': + [countQuery, searchQuery].forEach((query) => query?.whereIn(`${executionsTable}.arn`, value)); + break; case 'providerName': [countQuery, searchQuery].forEach((query) => query?.whereIn(`${providersTable}.name`, value)); break; @@ -347,7 +371,7 @@ class BaseSearch { break; case 'error.Error': [countQuery, searchQuery] - .forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' in ('${value.join('\',\'')}')`)); + .forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' in (${value.map(() => '?').join(',')})`, [...value])); break; case 'asyncOperationId': [countQuery, searchQuery].forEach((query) => query?.whereIn(`${asyncOperationsTable}.id`, value)); @@ -395,6 +419,9 @@ class BaseSearch { } Object.entries(omit(term, ['collectionName', 'collectionVersion'])).forEach(([name, value]) => { switch (name) { + case 'executionArn': + [countQuery, searchQuery].forEach((query) => query?.whereNot(`${executionsTable}.arn`, value)); + break; case 'providerName': [countQuery, searchQuery].forEach((query) => query?.whereNot(`${providersTable}.name`, value)); break; @@ -408,7 +435,7 @@ class BaseSearch { [countQuery, searchQuery].forEach((query) => query?.whereNot(`${executionsTable}_parent.arn`, value)); break; case 'error.Error': - [countQuery, searchQuery].forEach((query) => query?.whereRaw(`${this.tableName}.error->>'Error' != '${value}'`)); + [countQuery, searchQuery].forEach((query) => value && query?.whereRaw(`${this.tableName}.error->>'Error' != ?`, value)); break; default: [countQuery, searchQuery].forEach((query) => query?.whereNot(`${this.tableName}.${name}`, value)); @@ -432,7 +459,13 @@ class BaseSearch { const { sort } = dbQueryParameters || this.dbQueryParameters; sort?.forEach((key) => { if (key.column.startsWith('error')) { - searchQuery.orderByRaw(`${this.tableName}.error ->> 'Error' ${key.order}`); + searchQuery.orderByRaw( + `${this.tableName}.error ->> 'Error' ${key.order}` + ); + } else if (dbQueryParameters?.collate) { + searchQuery.orderByRaw( + `${key} collate \"${dbQueryParameters.collate}\"` + ); } else { searchQuery.orderBy([key]); } @@ -464,7 +497,7 @@ class BaseSearch { tableName? : string, }) : Promise { const { knex, tableName = this.tableName } = params; - const query = knex.raw(`EXPLAIN (FORMAT JSON) select * from "${tableName}"`); + const query = knex.raw('EXPLAIN (FORMAT JSON) select * from ??', tableName); log.debug(`Estimating the row count ${query.toSQL().sql}`); const countResult = await query; const countPath = 'rows[0]["QUERY PLAN"][0].Plan["Plan Rows"]'; diff --git a/packages/db/src/search/CollectionSearch.ts b/packages/db/src/search/CollectionSearch.ts index af30b66989b..a1d891d0aac 100644 --- a/packages/db/src/search/CollectionSearch.ts +++ b/packages/db/src/search/CollectionSearch.ts @@ -1,9 +1,12 @@ import { Knex } from 'knex'; +import omitBy from 'lodash/omitBy'; import pick from 'lodash/pick'; import Logger from '@cumulus/logger'; import { CollectionRecord } from '@cumulus/types/api/collections'; import { BaseSearch } from './BaseSearch'; +import { convertQueryStringToDbQueryParameters } from './queries'; +import { GranuleSearch } from './GranuleSearch'; import { DbQueryParameters, QueryEvent } from '../types/search'; import { translatePostgresCollectionToApiCollection } from '../translate/collections'; import { PostgresCollectionRecord } from '../types/collection'; @@ -27,6 +30,10 @@ interface CollectionRecordApi extends CollectionRecord { stats?: Statuses, } +const granuleFields = ['createdAt', 'granuleId', 'timestamp', 'updatedAt']; +const isGranuleField = (_value: any, key: string): boolean => + granuleFields.includes(key.split('__')[0]); + /** * Class to build and execute db search query for collections */ @@ -39,26 +46,13 @@ export class CollectionSearch extends BaseSearch { super({ queryStringParameters }, 'collection'); this.active = (active === 'true'); this.includeStats = (includeStats === 'true'); - } - /** - * Build basic query - * - * @param knex - DB client - * @returns queries for getting count and search result - */ - protected buildBasicQuery(knex: Knex) - : { - countQuery: Knex.QueryBuilder, - searchQuery: Knex.QueryBuilder, - } { - const countQuery = knex(this.tableName) - .count('*'); - - const searchQuery = knex(this.tableName) - .select(`${this.tableName}.*`); - - return { countQuery, searchQuery }; + // for active collection search, omit the fields which are for searching granules + if (this.active) { + this.dbQueryParameters = convertQueryStringToDbQueryParameters( + this.type, omitBy(this.queryStringParameters, isGranuleField) + ); + } } /** @@ -80,48 +74,52 @@ export class CollectionSearch extends BaseSearch { [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`)); } if (prefix) { - [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${prefix}%`)); + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `${prefix}%`)); } } /** - * Build queries for range fields + * Build subquery for active collections + * The subquery will search granules * - * @param params - * @param params.knex - db client - * @param [params.countQuery] - query builder for getting count - * @param params.searchQuery - query builder for search - * @param [params.dbQueryParameters] - db query parameters + * @param knex - db client + * @returns granule query */ - protected buildRangeQuery(params: { - knex: Knex, - countQuery: Knex.QueryBuilder, - searchQuery: Knex.QueryBuilder, - dbQueryParameters?: DbQueryParameters, - }) { - if (!this.active) { - super.buildRangeQuery(params); - return; - } - + private buildSubQueryForActiveCollections(knex: Knex): Knex.QueryBuilder { const granulesTable = TableNames.granules; - const { knex, countQuery, searchQuery, dbQueryParameters } = params; - const { range = {} } = dbQueryParameters ?? this.dbQueryParameters; + const granuleSearch = new GranuleSearch({ queryStringParameters: this.queryStringParameters }); + const { countQuery: subQuery } = granuleSearch.buildSearchForActiveCollections(knex); + + subQuery + .clear('select') + .select(1) + .where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`)) + .limit(1); + return subQuery; + } - const subQuery = knex.select(1).from(granulesTable) - .where(`${granulesTable}.collection_cumulus_id`, knex.raw(`${this.tableName}.cumulus_id`)); + /** + * Build the search query + * + * @param knex - DB client + * @returns queries for getting count and search result + */ + protected buildSearch(knex: Knex) + : { + countQuery?: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const queries = super.buildSearch(knex); + if (!this.active) { + return queries; + } - Object.entries(range).forEach(([name, rangeValues]) => { - if (rangeValues.gte) { - subQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte); - } - if (rangeValues.lte) { - subQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte); - } - }); - subQuery.limit(1); + const subQuery = this.buildSubQueryForActiveCollections(knex); + const { countQuery, searchQuery } = queries; + [countQuery, searchQuery].forEach((query) => query?.whereExists(subQuery)); - [countQuery, searchQuery].forEach((query) => query.whereExists(subQuery)); + log.debug(`buildSearch returns countQuery: ${countQuery?.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`); + return { countQuery, searchQuery }; } /** @@ -134,22 +132,22 @@ export class CollectionSearch extends BaseSearch { private async retrieveGranuleStats(collectionCumulusIds: number[], knex: Knex) : Promise { const granulesTable = TableNames.granules; - const statsQuery = knex(granulesTable) + let statsQuery = knex(granulesTable); + + if (this.active) { + const granuleSearch = new GranuleSearch({ + queryStringParameters: this.queryStringParameters, + }); + const { countQuery } = granuleSearch.buildSearchForActiveCollections(knex); + statsQuery = countQuery.clear('select'); + } + + statsQuery .select(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`) .count('*') .groupBy(`${granulesTable}.collection_cumulus_id`, `${granulesTable}.status`) .whereIn(`${granulesTable}.collection_cumulus_id`, collectionCumulusIds); - if (this.active) { - Object.entries(this.dbQueryParameters?.range ?? {}).forEach(([name, rangeValues]) => { - if (rangeValues.gte) { - statsQuery.where(`${granulesTable}.${name}`, '>=', rangeValues.gte); - } - if (rangeValues.lte) { - statsQuery.where(`${granulesTable}.${name}`, '<=', rangeValues.lte); - } - }); - } log.debug(`retrieveGranuleStats statsQuery: ${statsQuery?.toSQL().sql}`); const results = await statsQuery; const reduced = results.reduce((acc, record) => { @@ -180,6 +178,8 @@ export class CollectionSearch extends BaseSearch { protected async translatePostgresRecordsToApiRecords(pgRecords: PostgresCollectionRecord[], knex: Knex): Promise[]> { log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + + const { fields } = this.dbQueryParameters; let statsRecords: StatsRecords; const cumulusIds = pgRecords.map((record) => record.cumulus_id); if (this.includeStats) { @@ -188,9 +188,7 @@ export class CollectionSearch extends BaseSearch { const apiRecords = pgRecords.map((record) => { const apiRecord: CollectionRecordApi = translatePostgresCollectionToApiCollection(record); - const apiRecordFinal = this.dbQueryParameters.fields - ? pick(apiRecord, this.dbQueryParameters.fields) - : apiRecord; + const apiRecordFinal = fields ? pick(apiRecord, fields) : apiRecord; if (statsRecords) { apiRecordFinal.stats = statsRecords[record.cumulus_id] ? statsRecords[record.cumulus_id] diff --git a/packages/db/src/search/ExecutionSearch.ts b/packages/db/src/search/ExecutionSearch.ts index 9dd5621933b..a9e49d4a118 100644 --- a/packages/db/src/search/ExecutionSearch.ts +++ b/packages/db/src/search/ExecutionSearch.ts @@ -33,9 +33,9 @@ export class ExecutionSearch extends BaseSearch { } /** - * check if joined async_ops table search is needed + * check if joined async_operations table search is needed * - * @returns whether collection search is needed + * @returns whether async_operations search is needed */ protected searchAsync(): boolean { const { not, term, terms } = this.dbQueryParameters; @@ -43,9 +43,9 @@ export class ExecutionSearch extends BaseSearch { } /** - * check if joined async_ops table search is needed + * check if joined parent execution table search is needed * - * @returns whether collection search is needed + * @returns whether parent execution search is needed */ protected searchParent(): boolean { const { not, term, terms } = this.dbQueryParameters; @@ -130,7 +130,7 @@ export class ExecutionSearch extends BaseSearch { [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.arn`, `%${infix}%`)); } if (prefix) { - [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.arn`, `%${prefix}%`)); + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.arn`, `${prefix}%`)); } } @@ -143,6 +143,7 @@ export class ExecutionSearch extends BaseSearch { protected translatePostgresRecordsToApiRecords(pgRecords: ExecutionRecord[]) : Partial[] { log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + const { fields } = this.dbQueryParameters; const apiRecords = pgRecords.map((executionRecord: ExecutionRecord) => { const { collectionName, collectionVersion, asyncOperationId, parentArn } = executionRecord; const collectionId = collectionName && collectionVersion @@ -153,9 +154,7 @@ export class ExecutionSearch extends BaseSearch { asyncOperationId, parentArn, }); - return this.dbQueryParameters.fields - ? pick(apiRecord, this.dbQueryParameters.fields) - : apiRecord; + return fields ? pick(apiRecord, fields) : apiRecord; }); return apiRecords; } diff --git a/packages/db/src/search/GranuleSearch.ts b/packages/db/src/search/GranuleSearch.ts index c1b98ced70d..1ff9a909435 100644 --- a/packages/db/src/search/GranuleSearch.ts +++ b/packages/db/src/search/GranuleSearch.ts @@ -11,18 +11,16 @@ import { DbQueryParameters, QueryEvent } from '../types/search'; import { PostgresGranuleRecord } from '../types/granule'; import { translatePostgresGranuleToApiGranuleWithoutDbQuery } from '../translate/granules'; import { TableNames } from '../tables'; +import { FilePgModel } from '../models/file'; +import { PostgresFileRecord } from '../types/file'; +import { getExecutionInfoByGranuleCumulusIds } from '../lib/execution'; const log = new Logger({ sender: '@cumulus/db/GranuleSearch' }); interface GranuleRecord extends BaseRecord, PostgresGranuleRecord { - cumulus_id: number, - updated_at: Date, - collection_cumulus_id: number, collectionName: string, collectionVersion: string, - pdr_cumulus_id: number, pdrName?: string, - provider_cumulus_id?: number, providerName?: string, } @@ -110,15 +108,66 @@ export class GranuleSearch extends BaseSearch { } } + /** + * Build the search query for active collections. + * If time params are specified the query will search granules that have been updated + * in that time frame. If granuleId or providerId are provided, it will filter those as well. + * + * @param knex - DB client + * @returns queries for getting count and search result + */ + public buildSearchForActiveCollections(knex: Knex) + : { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const { countQuery, searchQuery } = this.buildBasicQuery(knex); + this.buildTermQuery({ countQuery, searchQuery }); + this.buildTermsQuery({ countQuery, searchQuery }); + this.buildRangeQuery({ knex, countQuery, searchQuery }); + + log.debug(`buildSearchForActiveCollections returns countQuery: ${countQuery?.toSQL().sql}, searchQuery: ${searchQuery.toSQL().sql}`); + return { countQuery, searchQuery }; + } + /** * Translate postgres records to api records * * @param pgRecords - postgres records returned from query + * @param knex - DB client * @returns translated api records */ - protected translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[]) - : Partial[] { + protected async translatePostgresRecordsToApiRecords(pgRecords: GranuleRecord[], knex: Knex) + : Promise[]> { log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + + const { fields, includeFullRecord } = this.dbQueryParameters; + + const fileMapping: { [key: number]: PostgresFileRecord[] } = {}; + const executionMapping: { [key: number]: { url: string, granule_cumulus_id: number } } = {}; + const cumulusIds = pgRecords.map((record) => record.cumulus_id); + if (includeFullRecord) { + //get Files + const fileModel = new FilePgModel(); + const files = await fileModel.searchByGranuleCumulusIds(knex, cumulusIds); + files.forEach((file) => { + if (!(file.granule_cumulus_id in fileMapping)) { + fileMapping[file.granule_cumulus_id] = []; + } + fileMapping[file.granule_cumulus_id].push(file); + }); + + //get Executions + const executions = await getExecutionInfoByGranuleCumulusIds({ + knexOrTransaction: knex, + granuleCumulusIds: cumulusIds, + }); + executions.forEach((execution) => { + if (!(execution.granule_cumulus_id in executionMapping)) { + executionMapping[execution.granule_cumulus_id] = execution; + } + }); + } const apiRecords = pgRecords.map((item: GranuleRecord) => { const granulePgRecord = item; const collectionPgRecord = { @@ -126,14 +175,21 @@ export class GranuleSearch extends BaseSearch { name: item.collectionName, version: item.collectionVersion, }; + const executionUrls = executionMapping[item.cumulus_id]?.url + ? [{ url: executionMapping[item.cumulus_id].url }] + : []; const pdr = item.pdrName ? { name: item.pdrName } : undefined; const providerPgRecord = item.providerName ? { name: item.providerName } : undefined; + const fileRecords = fileMapping[granulePgRecord.cumulus_id] || []; const apiRecord = translatePostgresGranuleToApiGranuleWithoutDbQuery({ - granulePgRecord, collectionPgRecord, pdr, providerPgRecord, + granulePgRecord, + collectionPgRecord, + pdr, + providerPgRecord, + files: fileRecords, + executionUrls, }); - return this.dbQueryParameters.fields - ? pick(apiRecord, this.dbQueryParameters.fields) - : apiRecord; + return fields ? pick(apiRecord, fields) : apiRecord; }); return apiRecords; } diff --git a/packages/db/src/search/PdrSearch.ts b/packages/db/src/search/PdrSearch.ts new file mode 100644 index 00000000000..b0f53ae258d --- /dev/null +++ b/packages/db/src/search/PdrSearch.ts @@ -0,0 +1,128 @@ +import { Knex } from 'knex'; +import pick from 'lodash/pick'; + +import { ApiPdrRecord } from '@cumulus/types/api/pdrs'; +import Logger from '@cumulus/logger'; + +import { BaseRecord } from '../types/base'; +import { BaseSearch } from './BaseSearch'; +import { DbQueryParameters, QueryEvent } from '../types/search'; +import { PostgresPdrRecord } from '../types/pdr'; +import { translatePostgresPdrToApiPdrWithoutDbQuery } from '../translate/pdr'; +import { TableNames } from '../tables'; + +const log = new Logger({ sender: '@cumulus/db/PdrSearch' }); + +interface PdrRecord extends BaseRecord, PostgresPdrRecord { + collectionName: string, + collectionVersion: string, + executionArn?: string, + providerName: string, +} + +/** + * Class to build and execute db search query for PDRs + */ +export class PdrSearch extends BaseSearch { + constructor(event: QueryEvent) { + super(event, 'pdr'); + } + + /** + * Build basic query + * + * @param knex - DB client + * @returns queries for getting count and search result + */ + protected buildBasicQuery(knex: Knex) + : { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const { + collections: collectionsTable, + providers: providersTable, + executions: executionsTable, + } = TableNames; + const countQuery = knex(this.tableName) + .count('*'); + + const searchQuery = knex(this.tableName) + .select(`${this.tableName}.*`) + .select({ + providerName: `${providersTable}.name`, + collectionName: `${collectionsTable}.name`, + collectionVersion: `${collectionsTable}.version`, + executionArn: `${executionsTable}.arn`, + }) + .innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`) + .innerJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`); + + if (this.searchCollection()) { + countQuery.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`); + } + + if (this.searchProvider()) { + countQuery.innerJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`); + } + + if (this.searchExecution()) { + countQuery.innerJoin(executionsTable, `${this.tableName}.execution_cumulus_id`, `${executionsTable}.cumulus_id`); + searchQuery.innerJoin(executionsTable, `${this.tableName}.execution_cumulus_id`, `${executionsTable}.cumulus_id`); + } else { + searchQuery.leftJoin(executionsTable, `${this.tableName}.execution_cumulus_id`, `${executionsTable}.cumulus_id`); + } + + return { countQuery, searchQuery }; + } + + /** + * Build queries for infix and prefix + * + * @param params + * @param params.countQuery - query builder for getting count + * @param params.searchQuery - query builder for search + * @param [params.dbQueryParameters] - db query parameters + */ + protected buildInfixPrefixQuery(params: { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + dbQueryParameters?: DbQueryParameters, + }) { + const { countQuery, searchQuery, dbQueryParameters } = params; + const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters; + if (infix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`)); + } + if (prefix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `${prefix}%`)); + } + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres records returned from query + * @returns translated api records + */ + protected translatePostgresRecordsToApiRecords(pgRecords: PdrRecord[]) + : Partial[] { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + const { fields } = this.dbQueryParameters; + const apiRecords = pgRecords.map((item: PdrRecord) => { + const pdrPgRecord = item; + const collectionPgRecord = { + cumulus_id: item.collection_cumulus_id, + name: item.collectionName, + version: item.collectionVersion, + }; + const providerPgRecord = { name: item.providerName }; + const executionArn = item.executionArn; + const apiRecord = translatePostgresPdrToApiPdrWithoutDbQuery({ + pdrPgRecord, collectionPgRecord, executionArn, providerPgRecord, + }); + return fields ? pick(apiRecord, fields) : apiRecord; + }); + return apiRecords; + } +} diff --git a/packages/db/src/search/ProviderSearch.ts b/packages/db/src/search/ProviderSearch.ts new file mode 100644 index 00000000000..d6aa1c82a53 --- /dev/null +++ b/packages/db/src/search/ProviderSearch.ts @@ -0,0 +1,63 @@ +import { Knex } from 'knex'; +import pick from 'lodash/pick'; + +import Logger from '@cumulus/logger'; +import { ApiProvider } from '@cumulus/types/api/providers'; +import { BaseSearch } from './BaseSearch'; +import { DbQueryParameters, QueryEvent } from '../types/search'; +import { translatePostgresProviderToApiProvider } from '../translate/providers'; +import { PostgresProviderRecord } from '../types/provider'; + +const log = new Logger({ sender: '@cumulus/db/ProviderSearch' }); + +/** + * Class to build and execute db search query for collections + */ +export class ProviderSearch extends BaseSearch { + constructor(event: QueryEvent) { + const queryStringParameters = event.queryStringParameters || {}; + super({ queryStringParameters }, 'provider'); + } + + /** + * Build queries for infix and prefix + * + * @param params + * @param params.countQuery - query builder for getting count + * @param params.searchQuery - query builder for search + * @param [params.dbQueryParameters] - db query parameters + */ + protected buildInfixPrefixQuery(params: { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + dbQueryParameters?: DbQueryParameters, + }) { + const { countQuery, searchQuery, dbQueryParameters } = params; + const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters; + if (infix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`)); + } + if (prefix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `${prefix}%`)); + } + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres Provider records returned from query + * @returns translated api records + */ + protected async translatePostgresRecordsToApiRecords(pgRecords: PostgresProviderRecord[]) + : Promise[]> { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + const apiRecords = pgRecords.map((record) => { + const apiRecord: ApiProvider = translatePostgresProviderToApiProvider(record); + const apiRecordFinal = this.dbQueryParameters.fields + ? pick(apiRecord, this.dbQueryParameters.fields) + : apiRecord; + return apiRecordFinal; + }); + return apiRecords; + } +} diff --git a/packages/db/src/search/ReconciliationReportSearch.ts b/packages/db/src/search/ReconciliationReportSearch.ts new file mode 100644 index 00000000000..a14bb282426 --- /dev/null +++ b/packages/db/src/search/ReconciliationReportSearch.ts @@ -0,0 +1,88 @@ +import { Knex } from 'knex'; +import Logger from '@cumulus/logger'; +import pick from 'lodash/pick'; + +import { ApiReconciliationReportRecord } from '@cumulus/types/api/reconciliation_reports'; +import { BaseSearch } from './BaseSearch'; +import { DbQueryParameters, QueryEvent } from '../types/search'; +import { translatePostgresReconReportToApiReconReport } from '../translate/reconciliation_reports'; +import { PostgresReconciliationReportRecord } from '../types/reconciliation_report'; +import { TableNames } from '../tables'; + +const log = new Logger({ sender: '@cumulus/db/ReconciliationReportSearch' }); + +/** + * Class to build and execute db search query for granules + */ +export class ReconciliationReportSearch extends BaseSearch { + constructor(event: QueryEvent) { + super(event, 'reconciliationReport'); + } + + /** + * Build basic query + * + * @param knex - DB client + * @returns queries for getting count and search result + */ + protected buildBasicQuery(knex: Knex) + : { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const { + reconciliationReports: reconciliationReportsTable, + } = TableNames; + const countQuery = knex(this.tableName) + .count('*'); + + const searchQuery = knex(this.tableName) + .select(`${this.tableName}.*`) + .select({ + reconciliationReportsName: `${reconciliationReportsTable}.name`, + }); + return { countQuery, searchQuery }; + } + + /** + * Build queries for infix and prefix + * + * @param params + * @param params.countQuery - query builder for getting count + * @param params.searchQuery - query builder for search + * @param [params.dbQueryParameters] - db query parameters + */ + protected buildInfixPrefixQuery(params: { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + dbQueryParameters?: DbQueryParameters, + }) { + const { countQuery, searchQuery, dbQueryParameters } = params; + const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters; + if (infix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`)); + } + if (prefix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `${prefix}%`)); + } + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres records returned from query + * @returns translated api records + */ + protected translatePostgresRecordsToApiRecords(pgRecords: PostgresReconciliationReportRecord[]) + : Partial[] { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + const { fields } = this.dbQueryParameters; + + const apiRecords = pgRecords.map((pgRecord) => { + const apiRecord = translatePostgresReconReportToApiReconReport(pgRecord); + return fields ? pick(apiRecord, fields) : apiRecord; + }); + + return apiRecords; + } +} diff --git a/packages/db/src/search/RuleSearch.ts b/packages/db/src/search/RuleSearch.ts new file mode 100644 index 00000000000..6a5ca270917 --- /dev/null +++ b/packages/db/src/search/RuleSearch.ts @@ -0,0 +1,123 @@ +import { Knex } from 'knex'; +import pick from 'lodash/pick'; + +import Logger from '@cumulus/logger'; +import { RuleRecord } from '@cumulus/types/api/rules'; +import { BaseSearch } from './BaseSearch'; +import { DbQueryParameters, QueryEvent } from '../types/search'; +import { PostgresRuleRecord } from '../types/rule'; +import { translatePostgresRuleToApiRuleWithoutDbQuery } from '../translate/rules'; +import { TableNames } from '../tables'; + +const log = new Logger({ sender: '@cumulus/db/RuleSearch' }); + +interface RuleRecordWithExternals extends PostgresRuleRecord { + collectionName: string, + collectionVersion: string, + providerName?: string, +} + +/** + * Class to build and execute db search query for rules + */ +export class RuleSearch extends BaseSearch { + constructor(event: QueryEvent) { + super(event, 'rule'); + } + + /** + * Build basic query + * + * @param knex - DB client + * @returns queries for getting count and search result + */ + protected buildBasicQuery(knex: Knex): { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + } { + const { + collections: collectionsTable, + providers: providersTable, + } = TableNames; + + const countQuery = knex(this.tableName) + .count(`${this.tableName}.cumulus_id`); + + const searchQuery = knex(this.tableName) + .select(`${this.tableName}.*`) + .select({ + collectionName: `${collectionsTable}.name`, + collectionVersion: `${collectionsTable}.version`, + providerName: `${providersTable}.name`, + }); + + if (this.searchCollection()) { + searchQuery.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`); + countQuery.innerJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`); + } else { + searchQuery.leftJoin(collectionsTable, `${this.tableName}.collection_cumulus_id`, `${collectionsTable}.cumulus_id`); + } + + if (this.searchProvider()) { + searchQuery.innerJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`); + countQuery.innerJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`); + } else { + searchQuery.leftJoin(providersTable, `${this.tableName}.provider_cumulus_id`, `${providersTable}.cumulus_id`); + } + + return { countQuery, searchQuery }; + } + + /** + * Build queries for infix and prefix + * + * @param params + * @param params.countQuery - query builder for getting count + * @param params.searchQuery - query builder for search + * @param [params.dbQueryParameters] - db query parameters + */ + protected buildInfixPrefixQuery(params: { + countQuery: Knex.QueryBuilder, + searchQuery: Knex.QueryBuilder, + dbQueryParameters?: DbQueryParameters, + }) { + const { countQuery, searchQuery, dbQueryParameters } = params; + const { infix, prefix } = dbQueryParameters ?? this.dbQueryParameters; + if (infix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `%${infix}%`)); + } + if (prefix) { + [countQuery, searchQuery].forEach((query) => query.whereLike(`${this.tableName}.name`, `${prefix}%`)); + } + } + + /** + * Translate postgres records to api records + * + * @param pgRecords - postgres Rule records returned from query + * @param knex - knex for the translation method + * @returns translated api records + */ + protected async translatePostgresRecordsToApiRecords( + pgRecords: RuleRecordWithExternals[] + ): Promise[]> { + log.debug(`translatePostgresRecordsToApiRecords number of records ${pgRecords.length} `); + + const apiRecords = pgRecords.map(async (record) => { + const providerPgRecord = record.providerName ? { name: record.providerName } : undefined; + const collectionPgRecord = record.collectionName ? { + name: record.collectionName, + version: record.collectionVersion, + } : undefined; + const apiRecord = await translatePostgresRuleToApiRuleWithoutDbQuery( + record, + collectionPgRecord, + providerPgRecord + ); + return this.dbQueryParameters.fields + ? pick(apiRecord, this.dbQueryParameters.fields) + : apiRecord; + }); + return await Promise.all(apiRecords); + } +} diff --git a/packages/db/src/search/StatsSearch.ts b/packages/db/src/search/StatsSearch.ts index 59e19804291..9d31efa4c9e 100644 --- a/packages/db/src/search/StatsSearch.ts +++ b/packages/db/src/search/StatsSearch.ts @@ -59,6 +59,7 @@ const infixMapping: { [key: string]: string } = { providers: 'name', executions: 'arn', pdrs: 'name', + reconciliationReports: 'name', }; /** @@ -241,7 +242,7 @@ class StatsSearch extends BaseSearch { searchQuery.whereLike(`${this.tableName}.${fieldName}`, `%${infix}%`); } if (prefix) { - searchQuery.whereLike(`${this.tableName}.${fieldName}`, `%${prefix}%`); + searchQuery.whereLike(`${this.tableName}.${fieldName}`, `${prefix}%`); } } @@ -251,7 +252,6 @@ class StatsSearch extends BaseSearch { * @param params * @param params.searchQuery - the search query * @param [params.dbQueryParameters] - the db query parameters - * @returns the updated search query based on queryStringParams */ protected buildTermQuery(params: { searchQuery: Knex.QueryBuilder, @@ -264,7 +264,7 @@ class StatsSearch extends BaseSearch { searchQuery.whereRaw(`${this.tableName}.error ->> 'Error' is not null`); } - return super.buildTermQuery({ + super.buildTermQuery({ ...params, dbQueryParameters: { term: omit(term, 'error.Error') }, }); diff --git a/packages/db/src/search/field-mapping.ts b/packages/db/src/search/field-mapping.ts index 39fd2ef61ec..5ad62ae22ce 100644 --- a/packages/db/src/search/field-mapping.ts +++ b/packages/db/src/search/field-mapping.ts @@ -92,6 +92,9 @@ const asyncOperationMapping : { [key: string]: Function } = { id: (value?: string) => ({ id: value, }), + _id: (value?: string) => ({ + id: value, + }), operationType: (value?: string) => ({ operation_type: value, }), @@ -155,7 +158,6 @@ const collectionMapping : { [key: string]: Function } = { }), }; -// TODO add and verify all queryable fields for the following record types const executionMapping : { [key: string]: Function } = { arn: (value?: string) => ({ arn: value, @@ -205,12 +207,30 @@ const executionMapping : { [key: string]: Function } = { }; const pdrMapping : { [key: string]: Function } = { + address: (value?: string) => ({ + address: value, + }), createdAt: (value?: string) => ({ created_at: value && new Date(Number(value)), }), + duration: (value?: string) => ({ + duration: value && Number(value), + }), + originalUrl: (value?: string) => ({ + original_url: value, + }), + PANSent: (value?: string) => ({ + pan_sent: (value === 'true'), + }), + PANmessage: (value?: string) => ({ + pan_message: value, + }), pdrName: (value?: string) => ({ name: value, }), + progress: (value?: string) => ({ + progress: value && Number(value), + }), status: (value?: string) => ({ status: value, }), @@ -231,24 +251,63 @@ const pdrMapping : { [key: string]: Function } = { provider: (value?: string) => ({ providerName: value, }), + execution: (value?: string) => ({ + executionArn: value && value.split('/').pop(), + }), }; const providerMapping : { [key: string]: Function } = { + allowedRedirects: (value?: string) => ({ + allowed_redirects: value?.split(','), + }), + certificateUrl: (value?: string) => ({ + certificate_url: value, + }), + cmKeyId: (value?: string) => ({ + cm_key_id: value, + }), createdAt: (value?: string) => ({ created_at: value && new Date(Number(value)), }), id: (value?: string) => ({ name: value, }), + name: (value?: string) => ({ + name: value, + }), timestamp: (value?: string) => ({ updated_at: value && new Date(Number(value)), }), updatedAt: (value?: string) => ({ updated_at: value && new Date(Number(value)), }), + globalConnectionLimit: (value?: string) => ({ + global_connection_limit: value && Number(value), + }), + host: (value?: string) => ({ + host: value, + }), + password: (value?: string) => ({ + password: value, + }), + port: (value?: string) => ({ + port: value, + }), + privateKey: (value?: string) => ({ + private_key: value, + }), + protocol: (value?: string) => ({ + protocol: value, + }), + username: (value?: string) => ({ + username: value, + }), }; const ruleMapping : { [key: string]: Function } = { + arn: (value?: string) => ({ + arn: value, + }), createdAt: (value?: string) => ({ created_at: value && new Date(Number(value)), }), @@ -264,6 +323,24 @@ const ruleMapping : { [key: string]: Function } = { updatedAt: (value?: string) => ({ updated_at: value && new Date(Number(value)), }), + workflow: (value?: string) => ({ + workflow: value, + }), + logEventArn: (value?: string) => ({ + log_event_arn: value, + }), + executionNamePrefix: (value?: string) => ({ + execution_name_prefix: value, + }), + queueUrl: (value?: string) => ({ + queue_url: value, + }), + 'rule.type': (value?: string) => ({ + type: value, + }), + 'rule.value': (value?: string) => ({ + value: value, + }), // The following fields require querying other tables collectionId: (value?: string) => { const { name, version } = (value && deconstructCollectionId(value)) || {}; @@ -277,6 +354,33 @@ const ruleMapping : { [key: string]: Function } = { }), }; +const reconciliationReportMapping: { [key: string]: Function } = { + name: (value?: string) => ({ + name: value, + }), + type: (value?: string) => ({ + type: value, + }), + status: (value?: string) => ({ + status: value, + }), + location: (value?: string) => ({ + location: value, + }), + error: (value?: string) => ({ + error: value, + }), + createdAt: (value?: string) => ({ + created_at: value && new Date(Number(value)), + }), + updatedAt: (value?: string) => ({ + updated_at: value && new Date(Number(value)), + }), + timestamp: (value?: string) => ({ + updated_at: value && new Date(Number(value)), + }), +}; + // type and its mapping const supportedMappings: { [key: string]: any } = { granule: granuleMapping, @@ -286,6 +390,7 @@ const supportedMappings: { [key: string]: any } = { pdr: pdrMapping, provider: providerMapping, rule: ruleMapping, + reconciliationReport: reconciliationReportMapping, }; /** diff --git a/packages/db/src/search/queries.ts b/packages/db/src/search/queries.ts index 824064a52da..dd290973ca0 100644 --- a/packages/db/src/search/queries.ts +++ b/packages/db/src/search/queries.ts @@ -243,9 +243,10 @@ export const convertQueryStringToDbQueryParameters = ( const dbQueryParameters: DbQueryParameters = {}; dbQueryParameters.page = Number.parseInt(page ?? '1', 10); - dbQueryParameters.limit = Number.parseInt(limit ?? '10', 10); - dbQueryParameters.offset = (dbQueryParameters.page - 1) * dbQueryParameters.limit; - + if (limit !== null) { + dbQueryParameters.limit = Number.parseInt(limit ?? '10', 10); + dbQueryParameters.offset = (dbQueryParameters.page - 1) * dbQueryParameters.limit; + } if (typeof infix === 'string') dbQueryParameters.infix = infix; if (typeof prefix === 'string') dbQueryParameters.prefix = prefix; if (typeof fields === 'string') dbQueryParameters.fields = fields.split(','); @@ -262,7 +263,6 @@ export const convertQueryStringToDbQueryParameters = ( // for each search strategy, get all parameters and convert them to db parameters Object.keys(regexes).forEach((k: string) => { const matchedFields = fieldsList.filter((f) => f.name.match(regexes[k])); - if (matchedFields && matchedFields.length > 0 && convert[k]) { const queryParams = convert[k](type, matchedFields, regexes[k]); Object.assign(dbQueryParameters, queryParams); diff --git a/packages/db/src/tables.ts b/packages/db/src/tables.ts index 43208f48c16..75bf9058697 100644 --- a/packages/db/src/tables.ts +++ b/packages/db/src/tables.ts @@ -7,5 +7,6 @@ export enum TableNames { granulesExecutions = 'granules_executions', pdrs = 'pdrs', providers = 'providers', + reconciliationReports = 'reconciliation_reports', rules = 'rules' } diff --git a/packages/db/src/test-utils.ts b/packages/db/src/test-utils.ts index 889387bebee..ae57ccd6451 100644 --- a/packages/db/src/test-utils.ts +++ b/packages/db/src/test-utils.ts @@ -16,6 +16,7 @@ import { PostgresFile } from './types/file'; import { PostgresGranule } from './types/granule'; import { PostgresPdr } from './types/pdr'; import { PostgresProvider } from './types/provider'; +import { PostgresReconciliationReport } from './types/reconciliation_report'; import { PostgresRule } from './types/rule'; export const createTestDatabase = async (knex: Knex, dbName: string, dbUser: string) => { @@ -137,7 +138,7 @@ export const fakeAsyncOperationRecordFactory = ( ): PostgresAsyncOperation => ({ id: uuidv4(), description: cryptoRandomString({ length: 10 }), - operation_type: 'ES Index', + operation_type: 'Reconciliation Report', status: 'RUNNING', output: { test: 'output' }, task_arn: cryptoRandomString({ length: 3 }), @@ -146,9 +147,20 @@ export const fakeAsyncOperationRecordFactory = ( export const fakePdrRecordFactory = ( params: Partial -) => ({ +): Partial => ({ name: `pdr${cryptoRandomString({ length: 10 })}`, status: 'running', created_at: new Date(), ...params, }); + +export const fakeReconciliationReportRecordFactory = ( + params: Partial +): PostgresReconciliationReport => ({ + name: `reconReport${cryptoRandomString({ length: 10 })}`, + type: 'Granule Inventory', + status: 'Generated', + created_at: new Date(), + updated_at: new Date(), + ...params, +}); diff --git a/packages/db/src/translate/async_operations.ts b/packages/db/src/translate/async_operations.ts index f87255d14c7..f454fb14497 100644 --- a/packages/db/src/translate/async_operations.ts +++ b/packages/db/src/translate/async_operations.ts @@ -1,3 +1,4 @@ +import omit from 'lodash/omit'; import { toSnake } from 'snake-camel'; import { ApiAsyncOperation } from '@cumulus/types/api/async_operations'; import Logger from '@cumulus/logger'; @@ -38,7 +39,7 @@ export const translateApiAsyncOperationToPostgresAsyncOperation = ( record: ApiAsyncOperation ): PostgresAsyncOperation => { // fix for old implementation of async-operation output assignment - const translatedRecord = toSnake(record); + const translatedRecord = toSnake(omit(record, 'timestamp')); if (record.output === 'none') { delete translatedRecord.output; } else if (record.output !== undefined) { diff --git a/packages/db/src/translate/pdr.ts b/packages/db/src/translate/pdr.ts index c38a7c55e69..d2c010825e0 100644 --- a/packages/db/src/translate/pdr.ts +++ b/packages/db/src/translate/pdr.ts @@ -9,6 +9,8 @@ import { CollectionPgModel } from '../models/collection'; import { ExecutionPgModel } from '../models/execution'; import { ProviderPgModel } from '../models/provider'; import { PostgresPdr, PostgresPdrRecord } from '../types/pdr'; +import { PostgresCollectionRecord } from '../types/collection'; +import { PostgresProviderRecord } from '../types/provider'; /** * Generate a Postgres PDR record from a DynamoDB record. @@ -57,6 +59,45 @@ export const translateApiPdrToPostgresPdr = async ( return removeNilProperties(pdrRecord); }; +/** + * Generate an API PDR object from the PDR and associated Postgres objects without + * querying the database + * + * @param params - params + * @param params.pdrPgRecord - PDR from Postgres + * @param params.collectionPgRecord - Collection from Postgres + * @param [params.executionArn] - executionUrl from Postgres + * @param [params.providerPgRecord] - provider from Postgres + * @returns An API PDR + */ +export const translatePostgresPdrToApiPdrWithoutDbQuery = ({ + pdrPgRecord, + collectionPgRecord, + executionArn, + providerPgRecord, +}: { + pdrPgRecord: PostgresPdrRecord, + collectionPgRecord: Pick, + executionArn?: string, + providerPgRecord: Pick, +}): ApiPdr => removeNilProperties({ + pdrName: pdrPgRecord.name, + provider: providerPgRecord?.name, + collectionId: constructCollectionId(collectionPgRecord.name, collectionPgRecord.version), + status: pdrPgRecord.status, + createdAt: pdrPgRecord.created_at.getTime(), + progress: pdrPgRecord.progress, + execution: executionArn ? getExecutionUrlFromArn(executionArn) : undefined, + PANSent: pdrPgRecord.pan_sent, + PANmessage: pdrPgRecord.pan_message, + stats: pdrPgRecord.stats, + address: pdrPgRecord.address, + originalUrl: pdrPgRecord.original_url, + timestamp: (pdrPgRecord.timestamp ? pdrPgRecord.timestamp.getTime() : undefined), + duration: pdrPgRecord.duration, + updatedAt: pdrPgRecord.updated_at.getTime(), +}); + /** * Generate a Postgres PDR record from a DynamoDB record. * @@ -85,23 +126,10 @@ export const translatePostgresPdrToApiPdr = async ( cumulus_id: postgresPDR.execution_cumulus_id, }) : undefined; - const apiPdr: ApiPdr = { - pdrName: postgresPDR.name, - provider: provider.name, - collectionId: constructCollectionId(collection.name, collection.version), - status: postgresPDR.status, - createdAt: postgresPDR.created_at.getTime(), - progress: postgresPDR.progress, - execution: execution ? getExecutionUrlFromArn(execution.arn) : undefined, - PANSent: postgresPDR.pan_sent, - PANmessage: postgresPDR.pan_message, - stats: postgresPDR.stats, - address: postgresPDR.address, - originalUrl: postgresPDR.original_url, - timestamp: (postgresPDR.timestamp ? postgresPDR.timestamp.getTime() : undefined), - duration: postgresPDR.duration, - updatedAt: postgresPDR.updated_at.getTime(), - }; - - return removeNilProperties(apiPdr); + return translatePostgresPdrToApiPdrWithoutDbQuery({ + pdrPgRecord: postgresPDR, + collectionPgRecord: collection, + executionArn: execution?.arn, + providerPgRecord: provider, + }); }; diff --git a/packages/db/src/translate/reconciliation_reports.ts b/packages/db/src/translate/reconciliation_reports.ts new file mode 100644 index 00000000000..64ec486460e --- /dev/null +++ b/packages/db/src/translate/reconciliation_reports.ts @@ -0,0 +1,39 @@ +import { ApiReconciliationReportRecord } from '@cumulus/types/api/reconciliation_reports'; +import { PostgresReconciliationReport, PostgresReconciliationReportRecord } from '../types/reconciliation_report'; + +const { removeNilProperties } = require('@cumulus/common/util'); +const pick = require('lodash/pick'); + +/** + * Generate a PostgreSQL Reconciliation Report from an API record. + * + * @param record - an API reconciliation report record + * @returns a PostgreSQL reconciliation report + */ +export const translateApiReconReportToPostgresReconReport = ( + record: ApiReconciliationReportRecord +): PostgresReconciliationReport => { + const pgReconciliationReport: PostgresReconciliationReport = removeNilProperties({ + ...pick(record, ['name', 'type', 'status', 'location', 'error']), + created_at: (record.createdAt ? new Date(record.createdAt) : undefined), + updated_at: (record.updatedAt ? new Date(record.updatedAt) : undefined), + }); + return pgReconciliationReport; +}; + +/** + * Generate an API Reconciliation Report record from a PostgreSQL record. + * + * @param pgReconciliationReport - a PostgreSQL reconciliation report record + * @returns ApiReconciliationReportRecord - an API reconciliation report record + */ +export const translatePostgresReconReportToApiReconReport = ( + pgReconciliationReport: PostgresReconciliationReportRecord +): ApiReconciliationReportRecord => { + const apiReconciliationReport = removeNilProperties({ + ...pick(pgReconciliationReport, ['name', 'type', 'status', 'location', 'error']), + createdAt: pgReconciliationReport.created_at?.getTime(), + updatedAt: pgReconciliationReport.updated_at?.getTime(), + }); + return apiReconciliationReport; +}; diff --git a/packages/db/src/translate/rules.ts b/packages/db/src/translate/rules.ts index 8af101284ab..39dafbcae89 100644 --- a/packages/db/src/translate/rules.ts +++ b/packages/db/src/translate/rules.ts @@ -5,27 +5,21 @@ import { RuleRecord, Rule } from '@cumulus/types/api/rules'; import { CollectionPgModel } from '../models/collection'; import { ProviderPgModel } from '../models/provider'; import { PostgresRule, PostgresRuleRecord } from '../types/rule'; +import { PostgresProviderRecord } from '../types/provider'; +import { PostgresCollectionRecord } from '../types/collection'; -export const translatePostgresRuleToApiRule = async ( +export const translatePostgresRuleToApiRuleWithoutDbQuery = async ( pgRule: PostgresRuleRecord, - knex: Knex | Knex.Transaction, - collectionPgModel = new CollectionPgModel(), - providerPgModel = new ProviderPgModel() + collectionPgRecord?: Pick, + providerPgRecord?: Partial ): Promise => { - const provider = pgRule.provider_cumulus_id - ? await providerPgModel.get(knex, { cumulus_id: pgRule.provider_cumulus_id }) - : undefined; - const collection = pgRule.collection_cumulus_id - ? await collectionPgModel.get(knex, { cumulus_id: pgRule.collection_cumulus_id }) - : undefined; - const apiRule: RuleRecord = { name: pgRule.name, workflow: pgRule.workflow, - provider: provider ? provider.name : undefined, - collection: collection ? { - name: collection.name, - version: collection.version, + provider: providerPgRecord ? providerPgRecord.name : undefined, + collection: collectionPgRecord ? { + name: collectionPgRecord.name, + version: collectionPgRecord.version, } : undefined, rule: removeNilProperties({ type: pgRule.type, @@ -45,6 +39,26 @@ export const translatePostgresRuleToApiRule = async ( return removeNilProperties(apiRule); }; +export const translatePostgresRuleToApiRule = async ( + pgRule: PostgresRuleRecord, + knex: Knex | Knex.Transaction, + collectionPgModel = new CollectionPgModel(), + providerPgModel = new ProviderPgModel() +): Promise => { + const providerPgRecord = pgRule.provider_cumulus_id + ? await providerPgModel.get(knex, { cumulus_id: pgRule.provider_cumulus_id }) + : undefined; + const collectionPgRecord = pgRule.collection_cumulus_id + ? await collectionPgModel.get(knex, { cumulus_id: pgRule.collection_cumulus_id }) + : undefined; + + return translatePostgresRuleToApiRuleWithoutDbQuery( + pgRule, + collectionPgRecord, + providerPgRecord + ); +}; + /** * Generate a Postgres rule record from a DynamoDB record. * diff --git a/packages/db/src/types/reconciliation_report.ts b/packages/db/src/types/reconciliation_report.ts new file mode 100644 index 00000000000..bd6671f8016 --- /dev/null +++ b/packages/db/src/types/reconciliation_report.ts @@ -0,0 +1,34 @@ +import { + ReconciliationReportType, + ReconciliationReportStatus, +} from '@cumulus/types/api/reconciliation_reports'; + +/** + * PostgresReconciliationReport + * + * This interface describes a Reconciliation Report object in postgres compatible format that + * is ready for write to Cumulus's postgres database instance + */ + +export interface PostgresReconciliationReport { + name: string, + type: ReconciliationReportType, + status: ReconciliationReportStatus, + location?: string, + error?: object, + created_at?: Date, + updated_at?: Date, +} + +/** + * PostgresReconciliationReportRecord + * + * This interface describes a Reconciliation Report Record that has been retrieved from + * postgres for reading. It differs from the PostgresReconciliationReport interface in that + * it types the autogenerated/required fields in the Postgres database as required + */ +export interface PostgresReconciliationReportRecord extends PostgresReconciliationReport { + cumulus_id: number, + created_at: Date, + updated_at: Date +} diff --git a/packages/db/src/types/search.ts b/packages/db/src/types/search.ts index 8d129082544..5011167965b 100644 --- a/packages/db/src/types/search.ts +++ b/packages/db/src/types/search.ts @@ -2,14 +2,14 @@ export type QueryStringParameters = { field?: string, fields?: string, infix?: string, - limit?: string, + limit?: string | null, page?: string, order?: string, prefix?: string, includeFullRecord?: string, sort_by?: string, sort_key?: string[], - [key: string]: string | string[] | undefined, + [key: string]: string | string[] | undefined | null, }; export type QueryEvent = { @@ -29,6 +29,7 @@ export type SortType = { }; export type DbQueryParameters = { + collate?: string, fields?: string[], infix?: string, limit?: number, diff --git a/packages/db/tests/lib/test-collection.js b/packages/db/tests/lib/test-collection.js index eeed7fc7a17..928273b723c 100644 --- a/packages/db/tests/lib/test-collection.js +++ b/packages/db/tests/lib/test-collection.js @@ -5,56 +5,85 @@ const sinon = require('sinon'); const cryptoRandomString = require('crypto-random-string'); const { - destroyLocalTestDb, - generateLocalTestDb, - GranulePgModel, CollectionPgModel, + destroyLocalTestDb, fakeCollectionRecordFactory, fakeGranuleRecordFactory, + fakeProviderRecordFactory, + generateLocalTestDb, getCollectionsByGranuleIds, + getUniqueCollectionsByGranuleFilter, + GranulePgModel, migrationDir, + ProviderPgModel, } = require('../../dist'); -const testDbName = `collection_${cryptoRandomString({ length: 10 })}`; - -test.before(async (t) => { +test.beforeEach(async (t) => { + t.context.testDbName = `collection_${cryptoRandomString({ length: 10 })}`; const { knexAdmin, knex } = await generateLocalTestDb( - testDbName, + t.context.testDbName, migrationDir ); t.context.knexAdmin = knexAdmin; t.context.knex = knex; t.context.collectionPgModel = new CollectionPgModel(); + t.context.providerPgModel = new ProviderPgModel(); t.context.granulePgModel = new GranulePgModel(); -}); - -test.after.always(async (t) => { - await destroyLocalTestDb({ - ...t.context, - testDbName, - }); -}); -test('getCollectionsByGranuleIds() returns collections for given granule IDs', async (t) => { - const collection1 = fakeCollectionRecordFactory(); - const collection2 = fakeCollectionRecordFactory(); + t.context.oldTimeStamp = '1950-01-01T00:00:00Z'; + t.context.newTimeStamp = '2020-01-01T00:00:00Z'; - const pgCollections = await t.context.collectionPgModel.insert( + t.context.collections = Array.from({ length: 3 }, (_, index) => { + const name = `collection${index + 1}`; + return fakeCollectionRecordFactory({ name, version: '001' }); + }); + t.context.pgCollections = await t.context.collectionPgModel.insert( t.context.knex, - [collection1, collection2], + t.context.collections, '*' ); + t.context.providers = Array.from({ length: 2 }, (_, index) => { + const name = `provider${index + 1}`; + return fakeProviderRecordFactory({ name }); + }); + t.context.pgProviders = await t.context.providerPgModel.create( + t.context.knex, + t.context.providers + ); - const granules = [ - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[0].cumulus_id }), - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[1].cumulus_id }), + t.context.granules = [ + fakeGranuleRecordFactory({ + collection_cumulus_id: t.context.pgCollections[0].cumulus_id, + provider_cumulus_id: t.context.pgProviders[0].cumulus_id, + updated_at: t.context.oldTimeStamp, + }), + fakeGranuleRecordFactory({ + collection_cumulus_id: t.context.pgCollections[1].cumulus_id, + provider_cumulus_id: t.context.pgProviders[1].cumulus_id, + updated_at: t.context.oldTimeStamp, + }), + fakeGranuleRecordFactory({ + collection_cumulus_id: t.context.pgCollections[2].cumulus_id, + provider_cumulus_id: t.context.pgProviders[1].cumulus_id, + updated_at: t.context.newTimeStamp, + }), ]; + await t.context.granulePgModel.insert( t.context.knex, - granules + t.context.granules ); +}); +test.afterEach.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + }); +}); + +test('getCollectionsByGranuleIds() returns collections for given granule IDs', async (t) => { + const { pgCollections, granules } = t.context; const collections = await getCollectionsByGranuleIds( t.context.knex, granules.map((granule) => granule.granule_id) @@ -64,25 +93,17 @@ test('getCollectionsByGranuleIds() returns collections for given granule IDs', a }); test('getCollectionsByGranuleIds() only returns unique collections', async (t) => { - const collection1 = fakeCollectionRecordFactory(); - const collection2 = fakeCollectionRecordFactory(); - - const pgCollections = await t.context.collectionPgModel.insert( - t.context.knex, - [collection1, collection2], - '*' - ); - - const granules = [ - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[0].cumulus_id }), - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[1].cumulus_id }), - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[1].cumulus_id }), - ]; + const { pgCollections } = t.context; + const testGranule = fakeGranuleRecordFactory({ + collection_cumulus_id: pgCollections[1].cumulus_id, + }); await t.context.granulePgModel.insert( t.context.knex, - granules + [testGranule] ); + const granules = [...t.context.granules, testGranule]; + const collections = await getCollectionsByGranuleIds( t.context.knex, granules.map((granule) => granule.granule_id) @@ -92,21 +113,15 @@ test('getCollectionsByGranuleIds() only returns unique collections', async (t) = }); test.serial('getCollectionsByGranuleIds() retries on connection terminated unexpectedly error', async (t) => { - const { knex } = t.context; - const collection1 = fakeCollectionRecordFactory(); - const collection2 = fakeCollectionRecordFactory(); - - const pgCollections = await t.context.collectionPgModel.insert( - knex, - [collection1, collection2], - '*' + const { knex, pgCollections } = t.context; + const testGranule = fakeGranuleRecordFactory({ + collection_cumulus_id: pgCollections[1].cumulus_id, + }); + await t.context.granulePgModel.insert( + t.context.knex, + [testGranule] ); - - const granules = [ - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[0].cumulus_id }), - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[1].cumulus_id }), - fakeGranuleRecordFactory({ collection_cumulus_id: pgCollections[1].cumulus_id }), - ]; + const granules = [...t.context.granules, testGranule]; const knexStub = sinon.stub(knex, 'select').returns({ select: sinon.stub().returnsThis(), @@ -127,3 +142,100 @@ test.serial('getCollectionsByGranuleIds() retries on connection terminated unexp ); t.is(error.attemptNumber, 4); }); + +test('getUniqueCollectionsByGranuleFilter filters by startTimestamp', async (t) => { + const { knex } = t.context; + const params = { + startTimestamp: '2005-01-01T00:00:00Z', + knex, + }; + + const result = await getUniqueCollectionsByGranuleFilter(params); + t.is(result.length, 1); +}); + +test('getUniqueCollectionsByGranuleFilter filters by endTimestamp', async (t) => { + const { knex } = t.context; + const params = { + endTimestamp: '2005-01-01T00:00:00Z', + knex, + }; + const result = await getUniqueCollectionsByGranuleFilter(params); + t.is(result.length, 2); + t.is(result[0].name, 'collection1'); + t.is(result[1].name, 'collection2'); +}); + +test('getUniqueCollectionsByGranuleFilter filters by collectionIds', async (t) => { + const { knex } = t.context; + const params = { + collectionIds: ['collection1___001', 'collection2___001'], + knex, + }; + + const result = await getUniqueCollectionsByGranuleFilter(params); + t.is(result.length, 2); + t.is(result[0].name, 'collection1'); + t.is(result[0].version, '001'); + t.is(result[1].name, 'collection2'); + t.is(result[1].version, '001'); +}); + +test('getUniqueCollectionsByGranuleFilter filters by granuleIds', async (t) => { + const { knex, granules } = t.context; + const params = { + granuleIds: [granules[0].granule_id], + knex, + }; + + const result = await getUniqueCollectionsByGranuleFilter(params); + t.is(result.length, 1); + t.is(result[0].name, 'collection1'); + t.is(result[0].version, '001'); +}); + +test('getUniqueCollectionsByGranuleFilter filters by providers', async (t) => { + const { knex, providers } = t.context; + const params = { + providers: [providers[0].name], + knex, + }; + + const result = await getUniqueCollectionsByGranuleFilter(params); + t.is(result.length, 1); + t.is(result[0].name, 'collection1'); + t.is(result[0].version, '001'); +}); + +test('getUniqueCollectionsByGranuleFilter orders collections by name', async (t) => { + const { knex } = t.context; + const params = { + knex, + }; + + const result = await getUniqueCollectionsByGranuleFilter(params); + t.is(result.length, 3); + t.is(result[0].name, 'collection1'); + t.is(result[1].name, 'collection2'); + t.is(result[2].name, 'collection3'); +}); + +test('getUniqueCollectionsByGranuleFilter returns distinct collections', async (t) => { + const { knex } = t.context; + const params = { + knex, + }; + + const granule = fakeGranuleRecordFactory({ + collection_cumulus_id: t.context.pgCollections[0].cumulus_id, + provider_cumulus_id: t.context.pgProviders[0].cumulus_id, + updated_at: t.context.oldTimeStamp, + }); + await t.context.granulePgModel.insert( + t.context.knex, + [granule] + ); + + const result = await getUniqueCollectionsByGranuleFilter(params); + t.is(result.length, 3); +}); diff --git a/packages/db/tests/lib/test-granule.js b/packages/db/tests/lib/test-granule.js index c9e15269d7e..0425f360e09 100644 --- a/packages/db/tests/lib/test-granule.js +++ b/packages/db/tests/lib/test-granule.js @@ -189,14 +189,14 @@ test('upsertGranuleWithExecutionJoinRecord() handles multiple executions for a g } ); t.deepEqual( - await granulesExecutionsPgModel.search( + orderBy(await granulesExecutionsPgModel.search( knex, { granule_cumulus_id: granuleCumulusId } - ), - [executionCumulusId, secondExecutionCumulusId].map((executionId) => ({ + ), 'execution_cumulus_id'), + orderBy([executionCumulusId, secondExecutionCumulusId].map((executionId) => ({ granule_cumulus_id: granuleCumulusId, execution_cumulus_id: executionId, - })) + })), 'execution_cumulus_id') ); }); @@ -534,12 +534,12 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by single ); t.teardown(() => granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id })); - const record = await getGranulesByApiPropertiesQuery( + const record = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { collectionIds: collectionId, - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -592,13 +592,13 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by multipl granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id }) ))); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { collectionIds: [collectionId, collectionId2], }, - ['granule_id'] - ); + sortByFields: ['granule_id'], + }); t.deepEqual( [{ ...granules.find((granule) => granule.granule_id === granule1.granule_id), @@ -633,12 +633,12 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by single t.teardown(() => granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id })); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { granuleIds: [granule.granule_id], - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -679,12 +679,12 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by multipl granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id }) ))); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { granuleIds: [granules[0].granule_id, granules[1].granule_id], - } - ); + }, + }); t.deepEqual( [{ ...granules[0], @@ -722,12 +722,12 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by provide '*' ); t.teardown(() => granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id })); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { providerNames: [fakeProvider.name], - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -739,6 +739,53 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by provide ); }); +test.serial('getGranulesByApiPropertiesQuery returns results POSIX/ASCII sorted when collition is set to "C"', async (t) => { + const { + collectionCumulusId, + knex, + granulePgModel, + providerPgModel, + } = t.context; + + const fakeProvider = fakeProviderRecordFactory(); + const [provider] = await providerPgModel.create(knex, fakeProvider); + + const granules = await granulePgModel.insert( + knex, + [ + fakeGranuleRecordFactory({ + collection_cumulus_id: collectionCumulusId, + provider_cumulus_id: provider.cumulus_id, + status: 'completed', + granule_id: 'MYDGRANULE', + }), + fakeGranuleRecordFactory({ + collection_cumulus_id: collectionCumulusId, + provider_cumulus_id: provider.cumulus_id, + status: 'completed', + granule_id: 'lowerCaseGranuleShouldGoLast', + }), + ], + '*' + ); + t.teardown(() => Promise.all(granules.map( + (granule) => + granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id }) + ))); + const query = getGranulesByApiPropertiesQuery({ + knex, + searchParams: { + collate: 'C', + status: 'completed', + }, + sortByFields: ['granule_id'], + }); + const records = await query; + t.is(records.length, 2); + t.is(records[0].granule_id, 'MYDGRANULE'); + t.is(records[1].granule_id, 'lowerCaseGranuleShouldGoLast'); +}); + test.serial('getGranulesByApiPropertiesQuery returns correct granules by status', async (t) => { const { collectionCumulusId, @@ -771,12 +818,12 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by status' (granule) => granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id }) ))); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { status: 'completed', - } - ); + }, + }); t.is(records.length, 1); t.deepEqual( [{ @@ -810,14 +857,14 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by updated ); t.teardown(() => granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id })); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { updatedAtRange: { updatedAtFrom: updatedAt, }, - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -828,14 +875,14 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by updated records ); - const records2 = await getGranulesByApiPropertiesQuery( + const records2 = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { updatedAtRange: { updatedAtFrom: new Date(now - 1), }, - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -868,14 +915,14 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by updated ); t.teardown(() => granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id })); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { updatedAtRange: { updatedAtTo: updatedAt, }, - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -886,14 +933,14 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by updated records ); - const records2 = await getGranulesByApiPropertiesQuery( + const records2 = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { updatedAtRange: { updatedAtTo: new Date(now + 1), }, - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -926,15 +973,15 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by updated ); t.teardown(() => granulePgModel.delete(knex, { cumulus_id: granule.cumulus_id })); - const records = await getGranulesByApiPropertiesQuery( + const records = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { updatedAtRange: { updatedAtFrom: updatedAt, updatedAtTo: updatedAt, }, - } - ); + }, + }); t.deepEqual( [{ ...granule, @@ -945,15 +992,15 @@ test.serial('getGranulesByApiPropertiesQuery returns correct granules by updated records ); - const records2 = await getGranulesByApiPropertiesQuery( + const records2 = await getGranulesByApiPropertiesQuery({ knex, - { + searchParams: { updatedAtRange: { updatedAtFrom: new Date(now - 1), updatedAtTo: new Date(now + 1), }, - } - ); + }, + }); t.deepEqual( [{ ...granule, diff --git a/packages/db/tests/models/test-file-model.js b/packages/db/tests/models/test-file-model.js index af1ffec0ed7..1d90161c1df 100644 --- a/packages/db/tests/models/test-file-model.js +++ b/packages/db/tests/models/test-file-model.js @@ -1,6 +1,6 @@ const test = require('ava'); const cryptoRandomString = require('crypto-random-string'); - +const range = require('lodash/range'); const { CollectionPgModel, GranulePgModel, @@ -107,3 +107,136 @@ test('FilePgModel.upsert() overwrites a file record', async (t) => { updatedFile ); }); + +test('FilePgModel.searchByGranuleCumulusIds() returns relevant files', async (t) => { + const usedGranuleCumulusIds = await Promise.all(range(5).map(() => ( + createFakeGranule(t.context.knex) + ))); + const unUsedGranuleCumulusIds = await Promise.all(range(5).map(() => ( + createFakeGranule(t.context.knex) + ))); + const relevantFiles = await t.context.filePgModel.insert( + t.context.knex, + usedGranuleCumulusIds.map((granuleCumulusId) => ( + fakeFileRecordFactory({ + granule_cumulus_id: granuleCumulusId, + }) + )) + ); + const irrelevantFiles = await t.context.filePgModel.insert( + t.context.knex, + unUsedGranuleCumulusIds.map((granuleCumulusId) => ( + fakeFileRecordFactory({ + granule_cumulus_id: granuleCumulusId, + }) + )) + ); + const searched = await t.context.filePgModel.searchByGranuleCumulusIds( + t.context.knex, + usedGranuleCumulusIds + ); + + const foundFileCumulusIds = searched.map((file) => file.cumulus_id); + const foundGranuleCumulusIds = searched.map((file) => file.granule_cumulus_id); + relevantFiles.forEach((relevantFile) => { + t.true(foundFileCumulusIds.includes(relevantFile.cumulus_id)); + }); + irrelevantFiles.forEach((irrelevantFile) => { + t.false(foundFileCumulusIds.includes(irrelevantFile.cumulus_id)); + }); + usedGranuleCumulusIds.forEach((usedGranuleCumulusId) => { + t.true(foundGranuleCumulusIds.includes(usedGranuleCumulusId)); + }); + unUsedGranuleCumulusIds.forEach((unUsedGranuleCumulusId) => { + t.false(foundGranuleCumulusIds.includes(unUsedGranuleCumulusId)); + }); +}); + +test('FilePgModel.searchByGranuleCumulusIds() allows to specify desired columns', async (t) => { + const usedGranuleCumulusIds = await Promise.all(range(5).map(() => ( + createFakeGranule(t.context.knex) + ))); + const unUsedGranuleCumulusIds = await Promise.all(range(5).map(() => ( + createFakeGranule(t.context.knex) + ))); + const relevantFiles = await t.context.filePgModel.insert( + t.context.knex, + usedGranuleCumulusIds.map((granuleCumulusId) => ( + fakeFileRecordFactory({ + granule_cumulus_id: granuleCumulusId, + }) + )) + ); + const irrelevantFiles = await t.context.filePgModel.insert( + t.context.knex, + unUsedGranuleCumulusIds.map((granuleCumulusId) => ( + fakeFileRecordFactory({ + granule_cumulus_id: granuleCumulusId, + }) + )) + ); + let searched = await t.context.filePgModel.searchByGranuleCumulusIds( + t.context.knex, + usedGranuleCumulusIds, + 'cumulus_id' + ); + + searched.forEach((file) => { + t.true(file.granule_cumulus_id === undefined); + t.true(file.created_at === undefined); + t.true(file.updated_at === undefined); + t.true(file.file_size === undefined); + t.true(file.bucket === undefined); + t.true(file.checksum_type === undefined); + t.true(file.checksum_value === undefined); + t.true(file.file_name === undefined); + t.true(file.key === undefined); + t.true(file.path === undefined); + t.true(file.source === undefined); + t.true(file.type === undefined); + }); + + let foundFileCumulusIds = searched.map((file) => file.cumulus_id); + relevantFiles.forEach((relevantFile) => { + t.true(foundFileCumulusIds.includes(relevantFile.cumulus_id)); + }); + irrelevantFiles.forEach((irrelevantFile) => { + t.false(foundFileCumulusIds.includes(irrelevantFile.cumulus_id)); + }); + + searched = await t.context.filePgModel.searchByGranuleCumulusIds( + t.context.knex, + usedGranuleCumulusIds, + ['cumulus_id', 'granule_cumulus_id'] + ); + + searched.forEach((file) => { + t.true(file.created_at === undefined); + t.true(file.updated_at === undefined); + t.true(file.file_size === undefined); + t.true(file.bucket === undefined); + t.true(file.checksum_type === undefined); + t.true(file.checksum_value === undefined); + t.true(file.file_name === undefined); + t.true(file.key === undefined); + t.true(file.path === undefined); + t.true(file.source === undefined); + t.true(file.type === undefined); + }); + + foundFileCumulusIds = searched.map((file) => file.cumulus_id); + const foundGranuleCumulusIds = searched.map((file) => file.granule_cumulus_id); + relevantFiles.forEach((relevantFile) => { + t.true(foundFileCumulusIds.includes(relevantFile.cumulus_id)); + }); + irrelevantFiles.forEach((irrelevantFile) => { + t.false(foundFileCumulusIds.includes(irrelevantFile.cumulus_id)); + }); + + usedGranuleCumulusIds.forEach((usedGranuleCumulusId) => { + t.true(foundGranuleCumulusIds.includes(usedGranuleCumulusId)); + }); + unUsedGranuleCumulusIds.forEach((unUsedGranuleCumulusId) => { + t.false(foundGranuleCumulusIds.includes(unUsedGranuleCumulusId)); + }); +}); diff --git a/packages/db/tests/models/test-reconciliation-report-model.js b/packages/db/tests/models/test-reconciliation-report-model.js new file mode 100644 index 00000000000..fe3c11a19c7 --- /dev/null +++ b/packages/db/tests/models/test-reconciliation-report-model.js @@ -0,0 +1,76 @@ +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); + +const { + ReconciliationReportPgModel, + fakeReconciliationReportRecordFactory, + generateLocalTestDb, + destroyLocalTestDb, + migrationDir, +} = require('../../dist'); + +const testDbName = `rule_${cryptoRandomString({ length: 10 })}`; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); +}); + +test.beforeEach((t) => { + t.context.reconciliationReportRecord = fakeReconciliationReportRecordFactory(); +}); + +test.after.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); +}); + +test('ReconciliationReportPgModel.upsert() creates new reconciliation report', async (t) => { + const { + knex, + reconciliationReportPgModel, + reconciliationReportRecord, + } = t.context; + + await reconciliationReportPgModel.upsert(knex, reconciliationReportRecord); + + const pgReport = await reconciliationReportPgModel.get(knex, reconciliationReportRecord); + t.like( + pgReport, + reconciliationReportRecord + ); +}); + +test('ReconciliationReportPgModel.upsert() overwrites a reconciliation report record', async (t) => { + const { + knex, + reconciliationReportPgModel, + reconciliationReportRecord, + } = t.context; + + await reconciliationReportPgModel.create(knex, reconciliationReportRecord); + + const updatedReconciliationReport = { + ...reconciliationReportRecord, + type: 'ORCA Backup', + status: 'Failed', + }; + + await reconciliationReportPgModel.upsert(knex, updatedReconciliationReport); + + const pgReport = await reconciliationReportPgModel.get(knex, { + name: reconciliationReportRecord.name, + }); + t.like( + pgReport, + updatedReconciliationReport + ); +}); diff --git a/packages/db/tests/search/test-AsyncOperationSearch.js b/packages/db/tests/search/test-AsyncOperationSearch.js new file mode 100644 index 00000000000..c46fda31caf --- /dev/null +++ b/packages/db/tests/search/test-AsyncOperationSearch.js @@ -0,0 +1,331 @@ +'use strict'; + +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); +const { v4: uuidv4 } = require('uuid'); +const omit = require('lodash/omit'); +const range = require('lodash/range'); +const { AsyncOperationSearch } = require('../../dist/search/AsyncOperationSearch'); +const { + translatePostgresAsyncOperationToApiAsyncOperation, +} = require('../../dist/translate/async_operations'); + +const { + destroyLocalTestDb, + generateLocalTestDb, + fakeAsyncOperationRecordFactory, + migrationDir, + AsyncOperationPgModel, +} = require('../../dist'); + +const testDbName = `asyncOperation_${cryptoRandomString({ length: 10 })}`; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + t.context.asyncOperationPgModel = new AsyncOperationPgModel(); + t.context.asyncOperations = []; + t.context.asyncOperationSearchTmestamp = 1579352700000; + + range(100).map((num) => ( + t.context.asyncOperations.push(fakeAsyncOperationRecordFactory({ + cumulus_id: num, + updated_at: new Date(t.context.asyncOperationSearchTmestamp + (num % 2)), + operation_type: num % 2 === 0 ? 'Bulk Granules' : 'Data Migration', + task_arn: num % 2 === 0 ? cryptoRandomString({ length: 3 }) : undefined, + })) + )); + + await t.context.asyncOperationPgModel.insert( + t.context.knex, + t.context.asyncOperations + ); +}); + +test.after.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); +}); + +test('AsyncOperationSearch returns 10 async operations by default', async (t) => { + const { knex } = t.context; + const dbSearch = new AsyncOperationSearch({}); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 100); + t.is(results.length, 10); +}); + +test('AsyncOperationSearch supports page and limit params', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 20, + page: 2, + }; + let dbSearch = new AsyncOperationSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 11, + page: 10, + }; + dbSearch = new AsyncOperationSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new AsyncOperationSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); + +test('AsyncOperationSearch supports infix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 20, + infix: t.context.asyncOperations[5].id.slice(1), + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test('AsyncOperationSearch supports prefix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 20, + prefix: t.context.asyncOperations[5].id.slice(0, -1), + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test('AsyncOperationSearch supports term search for uuid field', async (t) => { + const { knex } = t.context; + const dbRecord = t.context.asyncOperations[5]; + const queryStringParameters = { + limit: 200, + id: dbRecord.id, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test('AsyncOperationSearch supports term search for date field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + updatedAt: `${t.context.asyncOperationSearchTmestamp + 1}`, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test('AsyncOperationSearch supports term search for _id field', async (t) => { + const { knex } = t.context; + const dbRecord = t.context.asyncOperations[5]; + const queryStringParameters = { + limit: 200, + _id: dbRecord.id, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test('AsyncOperationSearch supports term search for string field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + operationType: 'Bulk Granules', + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test('AsyncOperationSearch supports range search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + timestamp__from: `${t.context.asyncOperationSearchTmestamp + 1}`, + timestamp__to: `${t.context.asyncOperationSearchTmestamp + 2}`, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test('AsyncOperationSearch supports search for multiple fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + id: t.context.asyncOperations[2].id, + updatedAt: `${t.context.asyncOperationSearchTmestamp}`, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); +}); + +test('AsyncOperationSearch non-existing fields are ignored', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 100); + t.is(results?.length, 100); +}); + +test('AsyncOperationSearch returns fields specified', async (t) => { + const { knex } = t.context; + const fields = 'id,operationType,status,taskArn'; + const queryStringParameters = { + fields, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 100); + t.is(results?.length, 10); + results.forEach((asyncOperation) => t.deepEqual(Object.keys(asyncOperation), fields.split(','))); +}); + +test('AsyncOperationSearch supports sorting', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + sort_by: 'id', + order: 'asc', + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.true(response.results[0].id < response.results[99].id); + t.true(response.results[0].id < response.results[50].id); + + queryStringParameters = { + limit: 200, + sort_key: ['-id'], + }; + const dbSearch2 = new AsyncOperationSearch({ queryStringParameters }); + const response2 = await dbSearch2.query(knex); + t.is(response2.meta.count, 100); + t.is(response2.results?.length, 100); + t.true(response2.results[0].id > response2.results[99].id); + t.true(response2.results[0].id > response2.results[50].id); + + queryStringParameters = { + limit: 200, + sort_by: 'operationType', + }; + const dbSearch3 = new AsyncOperationSearch({ queryStringParameters }); + const response3 = await dbSearch3.query(knex); + t.is(response3.meta.count, 100); + t.is(response3.results?.length, 100); + t.true(response3.results[0].operationType < response3.results[99].operationType); + t.true(response3.results[49].operationType < response3.results[50].operationType); +}); + +test('AsyncOperationSearch supports terms search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + operationType__in: ['Bulk Granules', 'NOTEXIST'].join(','), + }; + let dbSearch = new AsyncOperationSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + operationType__in: ['Bulk Granules', 'NOTEXIST'].join(','), + _id__in: [t.context.asyncOperations[2].id, uuidv4()].join(','), + }; + dbSearch = new AsyncOperationSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('AsyncOperationSearch supports search when asyncOperation field does not match the given value', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + operationType__not: 'Bulk Granules', + }; + let dbSearch = new AsyncOperationSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 200, + operationType__not: 'Bulk Granules', + id__not: t.context.asyncOperations[1].id, + }; + dbSearch = new AsyncOperationSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); +}); + +test('AsyncOperationSearch supports search which checks existence of asyncOperation field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + taskArn__exists: 'false', + output_exists: 'true', + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 50); + t.is(results?.length, 50); +}); + +test('AsyncOperationSearch returns the correct record', async (t) => { + const { knex } = t.context; + const dbRecord = t.context.asyncOperations[2]; + const queryStringParameters = { + limit: 200, + id: dbRecord.id, + }; + const dbSearch = new AsyncOperationSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); + + const expectedApiRecord = translatePostgresAsyncOperationToApiAsyncOperation(dbRecord); + t.deepEqual(omit(results?.[0], 'createdAt'), omit(expectedApiRecord, 'createdAt')); + t.truthy(results?.[0]?.createdAt); +}); diff --git a/packages/db/tests/search/test-CollectionSearch.js b/packages/db/tests/search/test-CollectionSearch.js index 595ebb81cb0..cf6e4c54be7 100644 --- a/packages/db/tests/search/test-CollectionSearch.js +++ b/packages/db/tests/search/test-CollectionSearch.js @@ -10,8 +10,10 @@ const { generateLocalTestDb, CollectionPgModel, GranulePgModel, + ProviderPgModel, fakeCollectionRecordFactory, fakeGranuleRecordFactory, + fakeProviderRecordFactory, migrationDir, } = require('../../dist'); @@ -27,11 +29,10 @@ test.before(async (t) => { t.context.knex = knex; t.context.collectionPgModel = new CollectionPgModel(); - const collections = []; t.context.collectionSearchTmestamp = 1579352700000; - range(100).map((num) => ( - collections.push(fakeCollectionRecordFactory({ + const collections = range(100).map((num) => ( + fakeCollectionRecordFactory({ name: num % 2 === 0 ? 'testCollection' : 'fakeCollection', version: num, cumulus_id: num, @@ -39,27 +40,38 @@ test.before(async (t) => { process: num % 2 === 0 ? 'ingest' : 'publish', report_to_ems: num % 2 === 0, url_path: num % 2 === 0 ? 'https://fakepath.com' : undefined, - })) + granule_id_validation_regex: num % 2 === 0 ? 'testGranuleId' : 'fakeGranuleId', + }) )); + // Create provider + t.context.providerPgModel = new ProviderPgModel(); + t.context.provider = fakeProviderRecordFactory(); + + const [pgProvider] = await t.context.providerPgModel.create( + t.context.knex, + t.context.provider + ); + t.context.providerCumulusId = pgProvider.cumulus_id; + t.context.granulePgModel = new GranulePgModel(); - const granules = []; const statuses = ['queued', 'failed', 'completed', 'running']; t.context.granuleSearchTmestamp = 1688888800000; - - range(1000).map((num) => ( - granules.push(fakeGranuleRecordFactory({ + t.context.granules = range(1000).map((num) => ( + fakeGranuleRecordFactory({ // collection with cumulus_id 0-9 each has 11 granules, // collection 10-98 has 10 granules, and collection 99 has 0 granule collection_cumulus_id: num % 99, cumulus_id: 100 + num, + // when collection_cumulus_id is odd number(1,3,5...97), its granules have provider + provider_cumulus_id: (num % 99 % 2) ? t.context.providerCumulusId : undefined, status: statuses[num % 4], // granule with collection_cumulus_id n has timestamp granuleSearchTmestamp + n, // except granule 98 (with collection 98 ) which has timestamp granuleSearchTmestamp - 1 updated_at: num === 98 ? new Date(t.context.granuleSearchTmestamp - 1) : new Date(t.context.granuleSearchTmestamp + (num % 99)), - })) + }) )); await t.context.collectionPgModel.insert( @@ -69,7 +81,7 @@ test.before(async (t) => { await t.context.granulePgModel.insert( t.context.knex, - granules + t.context.granules ); }); @@ -197,6 +209,15 @@ test('CollectionSearch supports term search for string field', async (t) => { const response3 = await dbSearch3.query(knex); t.is(response3.meta.count, 50); t.is(response3.results?.length, 50); + + queryStringParameters = { + limit: 200, + granuleId: 'testGranuleId', + }; + const dbSearch4 = new CollectionSearch({ queryStringParameters }); + const response4 = await dbSearch4.query(knex); + t.is(response4.meta.count, 50); + t.is(response4.results?.length, 50); }); test('CollectionSearch supports range search', async (t) => { @@ -320,6 +341,15 @@ test('CollectionSearch supports terms search', async (t) => { response = await dbSearch.query(knex); t.is(response.meta.count, 1); t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 200, + granuleId__in: ['testGranuleId', 'non-existent'].join(','), + }; + dbSearch = new CollectionSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); }); test('CollectionSearch supports search when collection field does not match the given value', async (t) => { @@ -396,6 +426,29 @@ test('CollectionSearch supports search for active collections', async (t) => { t.deepEqual(response.results[98].stats, expectedStats98); }); +test('CollectionSearch supports search for active collections by infix/prefix', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + infix: 'Collection', + prefix: 'fake', + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + + // collection_cumulus_id 1 + const expectedStats0 = { queued: 3, completed: 2, failed: 3, running: 3, total: 11 }; + // collection_cumulus_id 97 + const expectedStats48 = { queued: 3, completed: 2, failed: 3, running: 2, total: 10 }; + + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[48].stats, expectedStats48); +}); + test('CollectionSearch support search for active collections and stats with granules updated in the given time frame', async (t) => { const { knex } = t.context; const queryStringParameters = { @@ -409,13 +462,59 @@ test('CollectionSearch support search for active collections and stats with gran const dbSearch = new CollectionSearch({ queryStringParameters }); const response = await dbSearch.query(knex); - const expectedStats10 = { queued: 2, completed: 3, failed: 3, running: 2, total: 10 }; + const expectedStats0 = { queued: 2, completed: 3, failed: 3, running: 2, total: 10 }; // collection with cumulus_id 98 has 9 granules in the time frame const expectedStats98 = { queued: 2, completed: 2, failed: 3, running: 2, total: 9 }; // collections with cumulus_id 0-9 are filtered out t.is(response.meta.count, 89); t.is(response.results?.length, 89); - t.deepEqual(response.results[0].stats, expectedStats10); + t.deepEqual(response.results[0].stats, expectedStats0); t.deepEqual(response.results[88].stats, expectedStats98); }); + +test('CollectionSearch support search for active collections and stats with granules from a given provider', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + provider: t.context.provider.name, + sort_by: 'version', + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + + // collection_cumulus_id 1 + const expectedStats0 = { queued: 3, completed: 2, failed: 3, running: 3, total: 11 }; + // collection_cumulus_id 97 + const expectedStats48 = { queued: 3, completed: 2, failed: 3, running: 2, total: 10 }; + + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[48].stats, expectedStats48); +}); + +test('CollectionSearch support search for active collections and stats with granules in the granuleId list', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: '200', + active: 'true', + includeStats: 'true', + granuleId__in: [t.context.granules[0].granule_id, t.context.granules[5].granule_id].join(','), + sort_by: 'version', + }; + const dbSearch = new CollectionSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + + // collection_cumulus_id 0 + const expectedStats0 = { queued: 1, completed: 0, failed: 0, running: 0, total: 1 }; + // collection_cumulus_id 5 + const expectedStats1 = { queued: 0, completed: 0, failed: 1, running: 0, total: 1 }; + + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + t.deepEqual(response.results[0].stats, expectedStats0); + t.deepEqual(response.results[1].stats, expectedStats1); +}); diff --git a/packages/db/tests/search/test-GranuleSearch.js b/packages/db/tests/search/test-GranuleSearch.js index 5d055c47a53..29db5323e01 100644 --- a/packages/db/tests/search/test-GranuleSearch.js +++ b/packages/db/tests/search/test-GranuleSearch.js @@ -3,7 +3,7 @@ const cryptoRandomString = require('crypto-random-string'); const range = require('lodash/range'); const { constructCollectionId } = require('@cumulus/message/Collections'); - +const { sleep } = require('@cumulus/common'); const { CollectionPgModel, fakeCollectionRecordFactory, @@ -16,6 +16,11 @@ const { PdrPgModel, ProviderPgModel, migrationDir, + FilePgModel, + fakeFileRecordFactory, + ExecutionPgModel, + fakeExecutionRecordFactory, + GranulesExecutionsPgModel, } = require('../../dist'); const testDbName = `granule_${cryptoRandomString({ length: 10 })}`; @@ -139,7 +144,7 @@ test.before(async (t) => { ? t.context.granuleSearchFields.lastUpdateDateTime : undefined, published: !!(num % 2), product_volume: Math.round(Number(t.context.granuleSearchFields.productVolume) - * (1 / (num + 1))).toString(), + * (1 / (num + 1))).toString(), time_to_archive: !(num % 10) ? Number(t.context.granuleSearchFields.timeToArchive) : undefined, time_to_process: !(num % 20) @@ -148,6 +153,73 @@ test.before(async (t) => { updated_at: new Date(t.context.granuleSearchFields.timestamp + (num % 2) * 1000), })) ); + + const filePgModel = new FilePgModel(); + await filePgModel.insert( + knex, + t.context.pgGranules.map((granule) => fakeFileRecordFactory( + { + granule_cumulus_id: granule.cumulus_id, + path: 'a.txt', + checksum_type: 'md5', + } + )) + ); + await filePgModel.insert( + knex, + t.context.pgGranules.map((granule) => fakeFileRecordFactory( + { + granule_cumulus_id: granule.cumulus_id, + path: 'b.txt', + checksum_type: 'sha256', + } + )) + ); + + const executionPgModel = new ExecutionPgModel(); + const granuleExecutionPgModel = new GranulesExecutionsPgModel(); + + let executionRecords = await executionPgModel.insert( + knex, + t.context.pgGranules.map((_, i) => fakeExecutionRecordFactory({ + url: `earlierUrl${i}`, + })) + ); + await granuleExecutionPgModel.insert( + knex, + t.context.pgGranules.map((granule, i) => ({ + granule_cumulus_id: granule.cumulus_id, + execution_cumulus_id: executionRecords[i].cumulus_id, + })) + ); + executionRecords = []; + // it's important for later testing that these are uploaded strictly in order + for (const i of range(100)) { + const [executionRecord] = await executionPgModel.insert( // eslint-disable-line no-await-in-loop + knex, + [fakeExecutionRecordFactory({ + url: `laterUrl${i}`, + })] + ); + executionRecords.push(executionRecord); + //ensure that timestamp in execution record is distinct + await sleep(1); // eslint-disable-line no-await-in-loop + } + + await granuleExecutionPgModel.insert( + knex, + t.context.pgGranules.map((granule, i) => ({ + granule_cumulus_id: granule.cumulus_id, + execution_cumulus_id: executionRecords[i].cumulus_id, + })) + ); + await granuleExecutionPgModel.insert( + knex, + t.context.pgGranules.map((granule, i) => ({ + granule_cumulus_id: granule.cumulus_id, + execution_cumulus_id: executionRecords[99 - i].cumulus_id, + })) + ); }); test('GranuleSearch returns 10 granule records by default', async (t) => { @@ -859,3 +931,91 @@ test('GranuleSearch estimates the rowcount of the table by default', async (t) = t.true(response.meta.count > 0); t.is(response.results?.length, 50); }); + +test('GranuleSearch with includeFullRecord true retrieves associated file objects for granules', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.results?.length, 100); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.files?.length, 2); + t.true('bucket' in granuleRecord.files[0]); + t.true('key' in granuleRecord.files[0]); + t.true('bucket' in granuleRecord.files[1]); + t.true('key' in granuleRecord.files[1]); + }); +}); +test('GranuleSearch with includeFullRecord true retrieves associated file translated to api key format', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.results?.length, 100); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.files?.length, 2); + t.true('bucket' in granuleRecord.files[0]); + t.true('key' in granuleRecord.files[0]); + t.true('checksumType' in granuleRecord.files[0]); + t.true('bucket' in granuleRecord.files[1]); + t.true('key' in granuleRecord.files[1]); + t.true('checksumType' in granuleRecord.files[1]); + }); +}); + +test('GranuleSearch with includeFullRecord true retrieves one associated Url object for granules', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.results?.length, 100); + response.results.forEach((granuleRecord) => { + t.true('execution' in granuleRecord); + }); +}); + +test('GranuleSearch with includeFullRecord true retrieves latest associated Url object for granules', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.results?.length, 100); + response.results.sort((a, b) => a.cumulus_id - b.cumulus_id); + // these executions are loaded from lowest to highest number + // but each granule is associated with multiple executions: + // earlierUrl${i}, laterUrl${i}, and laterUrl${99-i} + // hence `laterUrl${max(i, 99-i)}` is the most recently updated execution + response.results.forEach((granuleRecord, i) => { + t.is(granuleRecord.execution, `laterUrl${Math.max(i, 99 - i)}`); + }); +}); + +test('GranuleSearch with includeFullRecord true retrieves granules, files and executions, with limit specifying number of granules', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 4, + includeFullRecord: 'true', + }; + const dbSearch = new GranuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.results?.length, 4); + response.results.forEach((granuleRecord) => { + t.is(granuleRecord.files?.length, 2); + t.true('bucket' in granuleRecord.files[0]); + t.true('key' in granuleRecord.files[0]); + t.true('bucket' in granuleRecord.files[1]); + t.true('key' in granuleRecord.files[1]); + }); +}); diff --git a/packages/db/tests/search/test-PdrSearch.js b/packages/db/tests/search/test-PdrSearch.js new file mode 100644 index 00000000000..f972f904f53 --- /dev/null +++ b/packages/db/tests/search/test-PdrSearch.js @@ -0,0 +1,751 @@ +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); + +const { constructCollectionId } = require('@cumulus/message/Collections'); + +const { + CollectionPgModel, + fakeCollectionRecordFactory, + fakeExecutionRecordFactory, + fakePdrRecordFactory, + fakeProviderRecordFactory, + generateLocalTestDb, + ExecutionPgModel, + PdrSearch, + PdrPgModel, + ProviderPgModel, + migrationDir, +} = require('../../dist'); + +const testDbName = `pdr_${cryptoRandomString({ length: 10 })}`; + +// generate PDR name for infix and prefix search +const generatePdrName = (num) => { + let name = cryptoRandomString({ length: 10 }); + if (num % 30 === 0) name = `${cryptoRandomString({ length: 5 })}infix${cryptoRandomString({ length: 5 })}`; + if (num % 25 === 0) name = `prefix${cryptoRandomString({ length: 10 })}`; + return name; +}; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + // Create collection + t.context.collectionPgModel = new CollectionPgModel(); + t.context.collectionName = 'fakeCollection'; + t.context.collectionVersion = 'v1'; + + const collectionName2 = 'testCollection2'; + const collectionVersion2 = 'v2'; + + t.context.collectionId = constructCollectionId( + t.context.collectionName, + t.context.collectionVersion + ); + + t.context.collectionId2 = constructCollectionId( + collectionName2, + collectionVersion2 + ); + + t.context.testPgCollection = fakeCollectionRecordFactory({ + name: t.context.collectionName, + version: t.context.collectionVersion, + }); + t.context.testPgCollection2 = fakeCollectionRecordFactory({ + name: collectionName2, + version: collectionVersion2, + }); + + const [pgCollection] = await t.context.collectionPgModel.create( + t.context.knex, + t.context.testPgCollection + ); + const [pgCollection2] = await t.context.collectionPgModel.create( + t.context.knex, + t.context.testPgCollection2 + ); + t.context.collectionCumulusId = pgCollection.cumulus_id; + t.context.collectionCumulusId2 = pgCollection2.cumulus_id; + + // Create provider + t.context.providerPgModel = new ProviderPgModel(); + t.context.provider = fakeProviderRecordFactory(); + + const [pgProvider] = await t.context.providerPgModel.create( + t.context.knex, + t.context.provider + ); + t.context.providerCumulusId = pgProvider.cumulus_id; + + // Create execution + t.context.executionPgModel = new ExecutionPgModel(); + t.context.execution = fakeExecutionRecordFactory(); + + const [pgExecution] = await t.context.executionPgModel.create( + t.context.knex, + t.context.execution + ); + t.context.executionCumulusId = pgExecution.cumulus_id; + + t.context.pdrSearchFields = { + createdAt: 1579352700000, + duration: 6.8, + progress: 0.9, + status: 'failed', + timestamp: 1579352700000, + updatedAt: 1579352700000, + }; + + t.context.pdrNames = range(100).map(generatePdrName); + t.context.pdrs = range(50).map((num) => fakePdrRecordFactory({ + name: t.context.pdrNames[num], + created_at: new Date(t.context.pdrSearchFields.createdAt), + collection_cumulus_id: (num % 2) + ? t.context.collectionCumulusId : t.context.collectionCumulusId2, + provider_cumulus_id: t.context.providerCumulusId, + execution_cumulus_id: !(num % 2) ? t.context.executionCumulusId : undefined, + status: !(num % 2) ? t.context.pdrSearchFields.status : 'completed', + progress: num / 50, + pan_sent: num % 2 === 0, + pan_message: `pan${cryptoRandomString({ length: 10 })}`, + stats: { + processing: 0, + completed: 0, + failed: 0, + total: 0, + }, + address: `address${cryptoRandomString({ length: 10 })}`, + original_url: !(num % 50) ? `url${cryptoRandomString({ length: 10 })}` : undefined, + duration: t.context.pdrSearchFields.duration + (num % 2), + updated_at: new Date(t.context.pdrSearchFields.timestamp + (num % 2) * 1000), + })); + + t.context.pdrPgModel = new PdrPgModel(); + t.context.pgPdrs = await t.context.pdrPgModel.insert( + knex, + t.context.pdrs + ); +}); + +test('PdrSearch returns 10 PDR records by default', async (t) => { + const { knex } = t.context; + const dbSearch = new PdrSearch(); + const response = await dbSearch.query(knex); + + t.is(response.meta.count, 50); + + const apiPdrs = response.results || {}; + t.is(apiPdrs.length, 10); + const validatedRecords = apiPdrs.filter((pdr) => ( + [t.context.collectionId, t.context.collectionId2].includes(pdr.collectionId) + && (pdr.provider === t.context.provider.name) + && (!pdr.execution || pdr.execution === t.context.execution.arn))); + t.is(validatedRecords.length, apiPdrs.length); +}); + +test('PdrSearch supports page and limit params', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 25, + page: 2, + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 10, + page: 5, + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 0); +}); + +test('PdrSearch supports infix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + infix: 'infix', + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('PdrSearch supports prefix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + prefix: 'prefix', + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); +}); + +test('PdrSearch supports collectionId term search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + collectionId: t.context.collectionId2, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports provider term search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + provider: t.context.provider.name, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('PdrSearch supports execution term search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + execution: `https://example.com/${t.context.execution.arn}`, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports term search for boolean field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + PANSent: 'true', + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports term search for date field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + updatedAt: `${t.context.pdrSearchFields.updatedAt}`, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports term search for number field', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + duration: t.context.pdrSearchFields.duration, + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + progress: t.context.pdrSearchFields.progress, + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('PdrSearch supports term search for string field', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + status: t.context.pdrSearchFields.status, + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + const dbRecord = t.context.pdrs[0]; + queryStringParameters = { + limit: 100, + address: dbRecord.address, + pdrName: dbRecord.name, + originalUrl: dbRecord.original_url, + PANmessage: dbRecord.pan_message, + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('PdrSearch supports term search for timestamp', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + timestamp: `${t.context.pdrSearchFields.timestamp}`, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports range search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + duration__from: `${t.context.pdrSearchFields.duration - 1}`, + duration__to: `${t.context.pdrSearchFields.duration + 1}`, + timestamp__from: `${t.context.pdrSearchFields.timestamp}`, + timestamp__to: `${t.context.pdrSearchFields.timestamp + 1600}`, + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 100, + timestamp__from: t.context.pdrSearchFields.timestamp, + timestamp__to: t.context.pdrSearchFields.timestamp + 500, + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + duration__from: `${t.context.pdrSearchFields.duration + 2}`, + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test('PdrSearch supports search for multiple fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + collectionId__in: [t.context.collectionId2, t.context.collectionId].join(','), + provider: t.context.provider.name, + PANSent__not: 'false', + status: 'failed', + timestamp__from: t.context.pdrSearchFields.timestamp, + timestamp__to: t.context.pdrSearchFields.timestamp + 500, + sort_key: ['collectionId', '-timestamp'], + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch non-existing fields are ignored', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('PdrSearch returns fields specified', async (t) => { + const { knex } = t.context; + const fields = 'pdrName,collectionId,progress,PANSent,status'; + const queryStringParameters = { + fields, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((pdr) => t.deepEqual(Object.keys(pdr), fields.split(','))); +}); + +test('PdrSearch supports sorting', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + sort_by: 'timestamp', + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results[0].updatedAt < response.results[25].updatedAt); + t.true(response.results[1].updatedAt < response.results[40].updatedAt); + + queryStringParameters = { + limit: 100, + sort_by: 'timestamp', + order: 'desc', + }; + const dbSearch2 = new PdrSearch({ queryStringParameters }); + const response2 = await dbSearch2.query(knex); + t.is(response2.meta.count, 50); + t.is(response2.results?.length, 50); + t.true(response2.results[0].updatedAt > response2.results[25].updatedAt); + t.true(response2.results[1].updatedAt > response2.results[40].updatedAt); + + queryStringParameters = { + limit: 100, + sort_key: ['-timestamp'], + }; + const dbSearch3 = new PdrSearch({ queryStringParameters }); + const response3 = await dbSearch3.query(knex); + t.is(response3.meta.count, 50); + t.is(response3.results?.length, 50); + t.true(response3.results[0].updatedAt > response3.results[25].updatedAt); + t.true(response3.results[1].updatedAt > response3.results[40].updatedAt); + + queryStringParameters = { + limit: 100, + sort_key: ['+progress'], + }; + const dbSearch4 = new PdrSearch({ queryStringParameters }); + const response4 = await dbSearch4.query(knex); + t.is(response4.meta.count, 50); + t.is(response4.results?.length, 50); + t.true(Number(response4.results[0].progress) < Number(response4.results[25].progress)); + t.true(Number(response4.results[1].progress) < Number(response4.results[40].progress)); + + queryStringParameters = { + limit: 100, + sort_key: ['-timestamp', '+progress'], + }; + const dbSearch5 = new PdrSearch({ queryStringParameters }); + const response5 = await dbSearch5.query(knex); + t.is(response5.meta.count, 50); + t.is(response5.results?.length, 50); + t.true(response5.results[0].updatedAt > response5.results[25].updatedAt); + t.true(response5.results[1].updatedAt > response5.results[40].updatedAt); + t.true(Number(response5.results[0].progress) < Number(response5.results[10].progress)); + t.true(Number(response5.results[30].progress) < Number(response5.results[40].progress)); + + queryStringParameters = { + limit: 100, + sort_key: ['-timestamp'], + sort_by: 'timestamp', + order: 'asc', + }; + const dbSearch6 = new PdrSearch({ queryStringParameters }); + const response6 = await dbSearch6.query(knex); + t.is(response6.meta.count, 50); + t.is(response6.results?.length, 50); + t.true(response6.results[0].updatedAt < response6.results[25].updatedAt); + t.true(response6.results[1].updatedAt < response6.results[40].updatedAt); +}); + +test('PdrSearch supports sorting by CollectionId', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + sort_by: 'collectionId', + order: 'asc', + }; + const dbSearch8 = new PdrSearch({ queryStringParameters }); + const response8 = await dbSearch8.query(knex); + t.is(response8.meta.count, 50); + t.is(response8.results?.length, 50); + t.true(response8.results[0].collectionId < response8.results[25].collectionId); + t.true(response8.results[1].collectionId < response8.results[40].collectionId); + + queryStringParameters = { + limit: 100, + sort_key: ['-collectionId'], + }; + const dbSearch9 = new PdrSearch({ queryStringParameters }); + const response9 = await dbSearch9.query(knex); + t.is(response9.meta.count, 50); + t.is(response9.results?.length, 50); + t.true(response9.results[0].collectionId > response9.results[25].collectionId); + t.true(response9.results[1].collectionId > response9.results[40].collectionId); +}); + +test('PdrSearch supports terms search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + pdrName__in: [t.context.pdrNames[0], t.context.pdrNames[5]].join(','), + PANSent__in: 'true,false', + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + + queryStringParameters = { + limit: 100, + pdrName__in: [t.context.pdrNames[0], t.context.pdrNames[5]].join(','), + PANSent__in: 'true', + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('PdrSearch supports collectionId terms search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + collectionId__in: [t.context.collectionId2, constructCollectionId('fakecollectionterms', 'v1')].join(','), + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + collectionId__in: [t.context.collectionId, t.context.collectionId2].join(','), + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('PdrSearch supports provider terms search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + provider__in: [t.context.provider.name, 'fakeproviderterms'].join(','), + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('PdrSearch supports execution terms search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + execution__in: [`https://example.con/${t.context.execution.arn}`, 'fakepdrterms'].join(','), + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports search when pdr field does not match the given value', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + pdrName__not: t.context.pdrNames[0], + PANSent__not: 'true', + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + pdrName__not: t.context.pdrNames[0], + PANSent__not: 'false', + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 24); + t.is(response.results?.length, 24); +}); + +test('PdrSearch supports search which collectionId does not match the given value', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + collectionId__not: t.context.collectionId2, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports search which provider does not match the given value', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + provider__not: t.context.provider.name, + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); + + queryStringParameters = { + limit: 100, + provider__not: 'providernotexist', + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('PdrSearch supports search which execution does not match the given value', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + execution__not: `https://example.com/${t.context.execution.arn}`, + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); + + queryStringParameters = { + limit: 100, + execution__not: 'executionnotexist', + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch supports search which checks existence of PDR field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + originalUrl__exists: 'true', + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('PdrSearch supports search which checks existence of collectionId', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + collectionId__exists: 'true', + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + queryStringParameters = { + limit: 100, + collectionId__exists: 'false', + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test('PdrSearch supports search which checks existence of provider', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + provider__exists: 'true', + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + + queryStringParameters = { + limit: 100, + provider__exists: 'false', + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 0); + t.is(response.results?.length, 0); +}); + +test('PdrSearch supports search which checks existence of execution', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 100, + execution__exists: 'true', + }; + let dbSearch = new PdrSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 100, + execution__exists: 'false', + }; + dbSearch = new PdrSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('PdrSearch returns the correct record', async (t) => { + const { knex } = t.context; + const dbRecord = t.context.pdrs[2]; + const queryStringParameters = { + limit: 100, + pdrName: dbRecord.name, + }; + const dbSearch = new PdrSearch({ queryStringParameters }); + const { results, meta } = await dbSearch.query(knex); + t.is(meta.count, 1); + t.is(results?.length, 1); + + const expectedApiRecord = { + pdrName: dbRecord.name, + provider: t.context.provider.name, + collectionId: t.context.collectionId2, + status: dbRecord.status, + createdAt: dbRecord.created_at.getTime(), + progress: dbRecord.progress, + execution: `https://console.aws.amazon.com/states/home?region=us-east-1#/executions/details/${t.context.execution.arn}`, + PANSent: dbRecord.pan_sent, + PANmessage: dbRecord.pan_message, + stats: { total: 0, failed: 0, completed: 0, processing: 0 }, + address: dbRecord.address, + duration: dbRecord.duration, + updatedAt: dbRecord.updated_at.getTime(), + }; + + t.deepEqual(results?.[0], expectedApiRecord); +}); diff --git a/packages/db/tests/search/test-ProviderSearch.js b/packages/db/tests/search/test-ProviderSearch.js new file mode 100644 index 00000000000..57d126bda33 --- /dev/null +++ b/packages/db/tests/search/test-ProviderSearch.js @@ -0,0 +1,317 @@ +'use strict'; + +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); +const { ProviderSearch } = require('../../dist/search/ProviderSearch'); + +const { + destroyLocalTestDb, + generateLocalTestDb, + ProviderPgModel, + fakeProviderRecordFactory, + migrationDir, +} = require('../../dist'); + +const testDbName = `provider_${cryptoRandomString({ length: 10 })}`; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + t.context.providerPgModel = new ProviderPgModel(); + t.context.providerSearchTimestamp = 1579352700000; + + const providers = range(100).map((num) => fakeProviderRecordFactory({ + cumulus_id: num, + updated_at: new Date(t.context.providerSearchTimestamp + (num % 2)), + created_at: new Date(t.context.providerSearchTimestamp - (num % 2)), + name: num % 2 === 0 ? `testProvider${num}` : `fakeProvider${num}`, + host: num % 2 === 0 ? 'cumulus-sit' : 'cumulus-uat', + global_connection_limit: num % 2 === 0 ? 0 : 10, + private_key: num % 2 === 0 ? `fakeKey${num}` : undefined, + })); + + await t.context.providerPgModel.insert(t.context.knex, providers); +}); + +test.after.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); +}); + +test('ProviderSearch returns 10 providers by default', async (t) => { + const { knex } = t.context; + const dbSearch = new ProviderSearch({}); + const results = await dbSearch.query(knex); + t.is(results.meta.count, 100); + t.is(results.results.length, 10); +}); + +test('ProviderSearch supports page and limit params', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 20, + page: 2, + }; + let dbSearch = new ProviderSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 20); + + queryStringParameters = { + limit: 11, + page: 10, + }; + dbSearch = new ProviderSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 1); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new ProviderSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 0); +}); + +test('ProviderSearch supports infix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 20, + infix: 'test', + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 20); + t.true(response.results?.every((provider) => provider.id.includes('test'))); +}); + +test('ProviderSearch supports prefix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 20, + prefix: 'fake', + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 20); + t.true(response.results?.every((provider) => provider.id.startsWith('fake'))); +}); + +test('ProviderSearch supports term search for date field', async (t) => { + const { knex } = t.context; + const testUpdatedAt = t.context.providerSearchTimestamp + 1; + const queryStringParameters = { + limit: 200, + updatedAt: `${testUpdatedAt}`, + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.updatedAt === testUpdatedAt)); +}); + +test('ProviderSearch supports term search for number field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + globalConnectionLimit: '10', + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.globalConnectionLimit === 10)); +}); + +test('ProviderSearch supports term search for string field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + host: 'cumulus-sit', + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.host === 'cumulus-sit')); +}); + +test('ProviderSearch supports range search', async (t) => { + const { knex } = t.context; + const timestamp1 = t.context.providerSearchTimestamp + 1; + const timestamp2 = t.context.providerSearchTimestamp + 2; + const queryStringParameters = { + limit: 200, + timestamp__from: `${timestamp1}`, + timestamp__to: `${timestamp2}`, + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.updatedAt >= timestamp1 + && provider.updatedAt <= timestamp2)); +}); + +test('ProviderSearch supports search for multiple fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + id: 'testProvider82', + host: 'cumulus-sit', + global_connection_limit: 0, + }; + + const expectedResponse = { + createdAt: 1579352700000, + host: 'cumulus-sit', + id: 'testProvider82', + globalConnectionLimit: 0, + privateKey: 'fakeKey82', + protocol: 's3', + updatedAt: 1579352700000, + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); + t.deepEqual(response.results[0], expectedResponse); +}); + +test('ProviderSearch non-existing fields are ignored', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); +}); + +test('ProviderSearch returns fields specified', async (t) => { + const { knex } = t.context; + let fields = 'id'; + let queryStringParameters = { + fields, + }; + let dbSearch = new ProviderSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 10); + response.results.forEach((provider) => t.deepEqual(Object.keys(provider), fields.split(','))); + + fields = 'id,host,globalConnectionLimit'; + queryStringParameters = { + fields, + }; + dbSearch = new ProviderSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 10); + response.results.forEach((provider) => t.deepEqual(Object.keys(provider), fields.split(','))); +}); + +test('ProviderSearch supports sorting', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + sort_by: 'id', + order: 'asc', + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 100); + t.is(response.results?.length, 100); + t.true(response.results[0].id < response.results[99].id); + t.true(response.results[0].id < response.results[50].id); + + queryStringParameters = { + limit: 200, + sort_key: ['-id'], + }; + const dbSearch2 = new ProviderSearch({ queryStringParameters }); + const response2 = await dbSearch2.query(knex); + t.is(response2.meta.count, 100); + t.is(response2.results?.length, 100); + t.true(response2.results[0].id > response2.results[99].id); + t.true(response2.results[0].id > response2.results[50].id); + + queryStringParameters = { + limit: 200, + sort_by: 'globalConnectionLimit', + }; + const dbSearch3 = new ProviderSearch({ queryStringParameters }); + const response3 = await dbSearch3.query(knex); + t.is(response3.meta.count, 100); + t.is(response3.results?.length, 100); + t.true(response3.results[0].globalConnectionLimit < response3.results[99].globalConnectionLimit); + t.true(response3.results[49].globalConnectionLimit < response3.results[50].globalConnectionLimit); +}); + +test('ProviderSearch supports terms search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + id__in: ['fakeProvider85', 'testProvider86'].join(','), + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 2); + t.is(response.results?.length, 2); + t.true(response.results?.every((provider) => ['fakeProvider85', 'testProvider86'].includes(provider.id))); +}); + +test('ProviderSearch supports search when provider field does not match the given value', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + host__not: 'cumulus-uat', + }; + let dbSearch = new ProviderSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.host !== 'cumulus-uat')); + + queryStringParameters = { + limit: 200, + host__not: 'cumulus-uat', + id__not: 'testProvider38', + }; + dbSearch = new ProviderSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 49); + t.is(response.results?.length, 49); + t.true(response.results?.every((provider) => provider.host !== 'cumulus-uat' && provider.id !== 'testProvider38')); +}); + +test('ProviderSearch supports search which checks existence of provider field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + privateKey__exists: 'true', + }; + const dbSearch = new ProviderSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((provider) => provider.privateKey)); +}); diff --git a/packages/db/tests/search/test-ReconciliationReportSearch.js b/packages/db/tests/search/test-ReconciliationReportSearch.js new file mode 100644 index 00000000000..e4a728e025a --- /dev/null +++ b/packages/db/tests/search/test-ReconciliationReportSearch.js @@ -0,0 +1,246 @@ +'use strict'; + +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); +const { ReconciliationReportSearch } = require('../../dist/search/ReconciliationReportSearch'); + +const { + ReconciliationReportPgModel, + fakeReconciliationReportRecordFactory, + generateLocalTestDb, + destroyLocalTestDb, + migrationDir, +} = require('../../dist'); + +const testDbName = `reconReport_${cryptoRandomString({ length: 10 })}`; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); + const reconReportTypes = ['Granule Inventory', 'Granule Not Found', 'Inventory', 'ORCA Backup']; + const reconReportStatuses = ['Generated', 'Pending', 'Failed']; + t.context.reconReportSearchTimestamp = 1704100000000; + t.context.reportBucket = cryptoRandomString({ length: 8 }); + t.context.reportKey = cryptoRandomString({ length: 8 }); + + const reconReports = range(50).map((num) => fakeReconciliationReportRecordFactory({ + name: `fakeReconReport-${num + 1}`, + type: reconReportTypes[num % 4], + status: reconReportStatuses[num % 3], + location: `s3://fakeBucket${t.context.reportBucket}/fakeKey${t.context.reportKey}`, + updated_at: new Date(t.context.reconReportSearchTimestamp + (num % 2)), + created_at: new Date(t.context.reconReportSearchTimestamp - (num % 2)), + })); + + await t.context.reconciliationReportPgModel.insert(t.context.knex, reconReports); +}); + +test.after.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); +}); + +test('ReconciliationReportSearch returns the correct response for a basic query', async (t) => { + const { knex } = t.context; + const dbSearch = new ReconciliationReportSearch({}); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results.length, 10); + + const expectedResponse1 = { + name: 'fakeReconReport-1', + type: 'Granule Inventory', + status: 'Generated', + location: `s3://fakeBucket${t.context.reportBucket}/fakeKey${t.context.reportKey}`, + updatedAt: t.context.reconReportSearchTimestamp, + createdAt: t.context.reconReportSearchTimestamp, + }; + + const expectedResponse10 = { + name: 'fakeReconReport-10', + type: 'Granule Not Found', + status: 'Generated', + location: `s3://fakeBucket${t.context.reportBucket}/fakeKey${t.context.reportKey}`, + updatedAt: t.context.reconReportSearchTimestamp + 1, + createdAt: t.context.reconReportSearchTimestamp - 1, + }; + + t.deepEqual(response.results[0], expectedResponse1); + t.deepEqual(response.results[9], expectedResponse10); +}); + +test('ReconciliationReportSearch supports page and limit params', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 25, + page: 2, + }; + let dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 10, + page: 5, + }; + dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 0); +}); + +test('ReconciliationReportSearch supports prefix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 50, + prefix: 'fakeReconReport-1', + }; + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 11); + t.is(response.results?.length, 11); +}); + +test('ReconciliationReportSearch supports infix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 50, + infix: 'conReport-2', + }; + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 11); + t.is(response.results?.length, 11); +}); + +test('ReconciliationReportSearch supports sorting', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + sort_by: 'type', + order: 'asc', + }; + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results[0].type < response.results[15].type); + t.true(response.results[16].type < response.results[30].type); + t.true(response.results[31].type < response.results[45].type); +}); + +test('ReconciliationReportSearch supports term search for string fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + status: 'Generated', + }; + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 17); + t.is(response.results?.length, 17); + t.true(response.results?.every((result) => result.status === 'Generated')); +}); + +test('ReconciliationReportSearch supports term search for date fields', async (t) => { + const { knex } = t.context; + const testUpdatedAt = t.context.reconReportSearchTimestamp + 1; + const queryStringParameters = { + limit: 100, + updatedAt: `${testUpdatedAt}`, + }; + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + t.true(response.results?.every((report) => report.updatedAt === testUpdatedAt)); +}); + +test('ReconciliationReportSearch supports range search', async (t) => { + const { knex } = t.context; + const timestamp1 = t.context.reconReportSearchTimestamp - 1; + const timestamp2 = t.context.reconReportSearchTimestamp + 1; + const queryStringParameters = { + limit: 100, + timestamp__from: `${timestamp1}`, + timestamp__to: `${timestamp2}`, + }; + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results?.every((report) => report.updatedAt >= timestamp1 + && report.updatedAt <= timestamp2)); +}); + +test('ReconciliationReportSearch supports search for multiple fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 50, + type: 'Inventory', + status: 'Failed', + }; + + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 4); + t.is(response.results?.length, 4); + t.true(response.results?.every((report) => + report.type === 'Inventory' && report.status === 'Failed')); +}); + +test('ReconciliationReportSearch returns fields specified', async (t) => { + const { knex } = t.context; + let fields = 'name'; + let queryStringParameters = { + fields, + }; + let dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((report) => t.deepEqual(Object.keys(report), fields.split(','))); + + fields = 'name,type,status'; + queryStringParameters = { + fields, + }; + dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((report) => t.deepEqual(Object.keys(report), fields.split(','))); +}); + +test('ReconciliationReportSearch ignores non-existing fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 100, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new ReconciliationReportSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); diff --git a/packages/db/tests/search/test-RuleSearch.js b/packages/db/tests/search/test-RuleSearch.js new file mode 100644 index 00000000000..d448a1d05b6 --- /dev/null +++ b/packages/db/tests/search/test-RuleSearch.js @@ -0,0 +1,433 @@ +'use strict'; + +const test = require('ava'); +const cryptoRandomString = require('crypto-random-string'); +const range = require('lodash/range'); +const { constructCollectionId } = require('@cumulus/message/Collections'); + +const { RuleSearch } = require('../../dist/search/RuleSearch'); + +const { + AsyncOperationPgModel, + CollectionPgModel, + destroyLocalTestDb, + fakeAsyncOperationRecordFactory, + fakeCollectionRecordFactory, + fakeRuleRecordFactory, + generateLocalTestDb, + migrationDir, + RulePgModel, + ProviderPgModel, + fakeProviderRecordFactory, +} = require('../../dist'); + +const testDbName = `rule_${cryptoRandomString({ length: 10 })}`; + +test.before(async (t) => { + const { knexAdmin, knex } = await generateLocalTestDb( + testDbName, + migrationDir + ); + + t.context.knexAdmin = knexAdmin; + t.context.knex = knex; + + // Create PG Collections + t.context.collectionPgModel = new CollectionPgModel(); + t.context.testPgCollection = fakeCollectionRecordFactory( + { cumulus_id: 0, + name: 'testCollection', + version: 8 } + ); + t.context.testPgCollection2 = fakeCollectionRecordFactory( + { cumulus_id: 1, + name: 'testCollection2', + version: 4 } + ); + + await t.context.collectionPgModel.insert( + t.context.knex, + t.context.testPgCollection + ); + + await t.context.collectionPgModel.insert( + t.context.knex, + t.context.testPgCollection2 + ); + + t.context.collectionCumulusId = t.context.testPgCollection.cumulus_id; + t.context.collectionCumulusId2 = t.context.testPgCollection2.cumulus_id; + + t.context.collectionId = constructCollectionId( + t.context.testPgCollection.name, + t.context.testPgCollection.version + ); + t.context.collectionId2 = constructCollectionId( + t.context.testPgCollection2.name, + t.context.testPgCollection2.version + ); + + // Create a Provider + t.context.providerPgModel = new ProviderPgModel(); + t.context.testProvider = fakeProviderRecordFactory({ + name: 'testProvider', + }); + t.context.testProvider2 = fakeProviderRecordFactory({ + name: 'testProvider2', + }); + + const [pgProvider] = await t.context.providerPgModel.insert( + t.context.knex, + t.context.testProvider + ); + const [pgProvider2] = await t.context.providerPgModel.insert( + t.context.knex, + t.context.testProvider2 + ); + + t.context.providerCumulusId = pgProvider.cumulus_id; + t.context.providerCumulusId2 = pgProvider2.cumulus_id; + + // Create an Async Operation + t.context.asyncOperationsPgModel = new AsyncOperationPgModel(); + t.context.testAsyncOperation = fakeAsyncOperationRecordFactory({ cumulus_id: 140 }); + t.context.asyncCumulusId = t.context.testAsyncOperation.cumulus_id; + + await t.context.asyncOperationsPgModel.insert( + t.context.knex, + t.context.testAsyncOperation + ); + + t.context.duration = 100; + + // Create a lot of Rules + t.context.ruleSearchFields = { + createdAt: new Date(2017, 11, 31), + updatedAt: new Date(2018, 0, 1), + updatedAt2: new Date(2018, 0, 2), + }; + t.context.rulePgModel = new RulePgModel(); + const rules = range(50).map((num) => fakeRuleRecordFactory({ + name: `fakeRule-${num}`, + created_at: t.context.ruleSearchFields.createdAt, + updated_at: (num % 2) ? + t.context.ruleSearchFields.updatedAt : t.context.ruleSearchFields.updatedAt2, + enabled: num % 2 === 0, + workflow: `testWorkflow-${num}`, + queue_url: (num % 2) ? 'https://sqs.us-east-1.amazonaws.com/123/456' : null, + collection_cumulus_id: (num % 2) + ? t.context.collectionCumulusId : t.context.collectionCumulusId2, + provider_cumulus_id: (num % 2) + ? t.context.providerCumulusId : t.context.providerCumulusId2, + })); + await t.context.rulePgModel.insert(t.context.knex, rules); +}); + +test.after.always(async (t) => { + await destroyLocalTestDb({ + ...t.context, + testDbName, + }); +}); + +test('RuleSearch returns the correct response for a basic query', async (t) => { + const { knex } = t.context; + const dbSearch = new RuleSearch({}); + const results = await dbSearch.query(knex); + t.is(results.meta.count, 50); + t.is(results.results.length, 10); + + const expectedResponse1 = { + name: 'fakeRule-0', + createdAt: t.context.ruleSearchFields.createdAt.getTime(), + updatedAt: t.context.ruleSearchFields.updatedAt2.getTime(), + state: 'ENABLED', + rule: { + type: 'onetime', + }, + workflow: 'testWorkflow-0', + collection: { + name: 'testCollection2', + version: '4', + }, + provider: t.context.testProvider2.name, + }; + + const expectedResponse10 = { + name: 'fakeRule-9', + createdAt: t.context.ruleSearchFields.createdAt.getTime(), + updatedAt: t.context.ruleSearchFields.updatedAt.getTime(), + state: 'DISABLED', + rule: { + type: 'onetime', + }, + workflow: 'testWorkflow-9', + collection: { + name: 'testCollection', + version: '8', + }, + provider: t.context.testProvider.name, + queueUrl: 'https://sqs.us-east-1.amazonaws.com/123/456', + }; + + t.deepEqual(results.results[0], expectedResponse1); + t.deepEqual(results.results[9], expectedResponse10); +}); + +test('RuleSearch supports page and limit params', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 25, + page: 2, + }; + let dbSearch = new RuleSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 10, + page: 5, + }; + dbSearch = new RuleSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + + queryStringParameters = { + limit: 10, + page: 11, + }; + dbSearch = new RuleSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 0); +}); + +test('RuleSearch supports infix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 50, + infix: 'Rule-27', + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('RuleSearch supports prefix search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 50, + prefix: 'fakeRule-1', + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 11); + t.is(response.results?.length, 11); +}); + +test('RuleSearch supports term search for string field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 10, + workflow: 'testWorkflow-11', + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 1); + t.is(response.results?.length, 1); +}); + +test('RuleSearch non-existing fields are ignored', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + non_existing_field: `non_exist_${cryptoRandomString({ length: 5 })}`, + non_existing_field__from: `non_exist_${cryptoRandomString({ length: 5 })}`, + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('RuleSearch returns fields specified', async (t) => { + const { knex } = t.context; + const fields = 'state,name'; + const queryStringParameters = { + fields, + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 10); + response.results.forEach((rule) => t.deepEqual(Object.keys(rule), fields.split(','))); +}); + +test('RuleSearch supports search for multiple fields', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 10, + prefix: 'fakeRule-1', + state: 'DISABLED', + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + + t.is(response.meta.count, 6); + t.is(response.results?.length, 6); +}); + +test('RuleSearch supports sorting', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + sort_by: 'workflow', + order: 'desc', + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); + t.true(response.results[0].workflow > response.results[10].workflow); + t.true(response.results[1].workflow > response.results[30].workflow); +}); + +test('RuleSearch supports collectionId term search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + collectionId: t.context.collectionId, + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('RuleSearch supports provider term search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + provider: t.context.testProvider.name, + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('RuleSearch supports term search for date field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + updatedAt: t.context.ruleSearchFields.updatedAt, + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('RuleSearch supports term search for boolean field', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + state: 'ENABLED', // maps to the bool field "enabled" + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('RuleSearch supports term search for timestamp', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + timestamp: t.context.ruleSearchFields.updatedAt, //maps to timestamp + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('RuleSearch supports range search', async (t) => { + const { knex } = t.context; + const queryStringParameters = { + limit: 200, + timestamp__from: t.context.ruleSearchFields.timestamp, + timestamp__to: t.context.ruleSearchFields.timestamp + 1600, + }; + const dbSearch = new RuleSearch({ queryStringParameters }); + const response = await dbSearch.query(knex); + + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('RuleSearch supports search which checks existence of queue URL field', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + queueUrl__exists: 'true', + }; + let dbSearch = new RuleSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 200, + queueUrl__exists: 'false', + }; + dbSearch = new RuleSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); +}); + +test('RuleSearch supports collectionId terms search', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + collectionId__in: [t.context.collectionId2, constructCollectionId('fakecollectionterms', 'v1')].join(','), + }; + let dbSearch = new RuleSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 200, + collectionId__in: [t.context.collectionId, t.context.collectionId2].join(','), + }; + dbSearch = new RuleSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); + +test('RuleSearch supports search which provider does not match the given value', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + limit: 200, + provider__not: t.context.testProvider.name, + }; + let dbSearch = new RuleSearch({ queryStringParameters }); + let response = await dbSearch.query(knex); + t.is(response.meta.count, 25); + t.is(response.results?.length, 25); + + queryStringParameters = { + limit: 200, + provider__not: 'providernotexist', + }; + dbSearch = new RuleSearch({ queryStringParameters }); + response = await dbSearch.query(knex); + t.is(response.meta.count, 50); + t.is(response.results?.length, 50); +}); diff --git a/packages/db/tests/search/test-StatsSearch.js b/packages/db/tests/search/test-StatsSearch.js index a2a7faba6dc..825d0f4af74 100644 --- a/packages/db/tests/search/test-StatsSearch.js +++ b/packages/db/tests/search/test-StatsSearch.js @@ -8,17 +8,19 @@ const { StatsSearch } = require('../../dist/search/StatsSearch'); const { destroyLocalTestDb, generateLocalTestDb, - GranulePgModel, CollectionPgModel, + GranulePgModel, + ExecutionPgModel, + PdrPgModel, + ProviderPgModel, + ReconciliationReportPgModel, fakeCollectionRecordFactory, fakeGranuleRecordFactory, + fakeExecutionRecordFactory, + fakePdrRecordFactory, fakeProviderRecordFactory, + fakeReconciliationReportRecordFactory, migrationDir, - fakePdrRecordFactory, - fakeExecutionRecordFactory, - PdrPgModel, - ExecutionPgModel, - ProviderPgModel, } = require('../../dist'); const testDbName = `collection_${cryptoRandomString({ length: 10 })}`; @@ -34,88 +36,67 @@ test.before(async (t) => { t.context.collectionPgModel = new CollectionPgModel(); t.context.granulePgModel = new GranulePgModel(); - t.context.providerPgModel = new ProviderPgModel(); - t.context.pdrPgModel = new PdrPgModel(); t.context.executionPgModel = new ExecutionPgModel(); + t.context.pdrPgModel = new PdrPgModel(); + t.context.providerPgModel = new ProviderPgModel(); + t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); const statuses = ['queued', 'failed', 'completed', 'running']; const errors = [{ Error: 'UnknownError' }, { Error: 'CumulusMessageAdapterError' }, { Error: 'IngestFailure' }, { Error: 'CmrFailure' }, {}]; - const granules = []; - const collections = []; - const executions = []; - const pdrs = []; - const providers = []; - - range(20).map((num) => ( - collections.push(fakeCollectionRecordFactory({ - name: 'testCollection', - version: `${num}`, - cumulus_id: num, - })) - )); - - range(10).map((num) => ( - providers.push(fakeProviderRecordFactory({ - cumulus_id: num, - name: `testProvider${num}`, - })) - )); - - range(100).map((num) => ( - granules.push(fakeGranuleRecordFactory({ - collection_cumulus_id: num % 20, - granule_id: num % 2 === 0 ? `testGranule${num}` : `query__Granule${num}`, - status: statuses[num % 4], - created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), - updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), - error: errors[num % 5], - duration: num + (num / 10), - provider_cumulus_id: num % 10, - })) - )); - - range(20).map((num) => ( - pdrs.push(fakePdrRecordFactory({ - collection_cumulus_id: num, - status: statuses[(num % 3) + 1], - provider_cumulus_id: num % 10, - created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), - updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), - // eslint-disable-next-line no-sequences - })), - executions.push(fakeExecutionRecordFactory({ - collection_cumulus_id: num, - status: statuses[(num % 3) + 1], - error: errors[num % 5], - created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), - updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), - })) - )); - - await t.context.collectionPgModel.insert( - t.context.knex, - collections - ); - - await t.context.providerPgModel.insert( - t.context.knex, - providers - ); - - await t.context.granulePgModel.insert( - t.context.knex, - granules - ); - - await t.context.executionPgModel.insert( - t.context.knex, - executions - ); - - await t.context.pdrPgModel.insert( - t.context.knex, - pdrs - ); + const reconReportTypes = ['Granule Inventory', 'Granule Not Found', 'Inventory', 'ORCA Backup']; + const reconReportStatuses = ['Generated', 'Pending', 'Failed']; + + const collections = range(20).map((num) => fakeCollectionRecordFactory({ + name: 'testCollection', + version: `${num}`, + cumulus_id: num, + })); + + const providers = range(10).map((num) => fakeProviderRecordFactory({ + cumulus_id: num, + name: `testProvider${num}`, + })); + + const granules = range(100).map((num) => fakeGranuleRecordFactory({ + collection_cumulus_id: num % 20, + granule_id: num % 2 === 0 ? `testGranule${num}` : `query__Granule${num}`, + status: statuses[num % 4], + created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + error: errors[num % 5], + duration: num + (num / 10), + provider_cumulus_id: num % 10, + })); + + const pdrs = range(20).map((num) => fakePdrRecordFactory({ + collection_cumulus_id: num, + status: statuses[(num % 3) + 1], + provider_cumulus_id: num % 10, + created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + })); + + const executions = range(20).map((num) => fakeExecutionRecordFactory({ + collection_cumulus_id: num, + status: statuses[(num % 3) + 1], + error: errors[num % 5], + created_at: (new Date(2018 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2018 + (num % 6), (num % 12), ((num + 1) % 29))), + })); + + const reconReports = range(24).map((num) => fakeReconciliationReportRecordFactory({ + type: reconReportTypes[(num % 4)], + status: reconReportStatuses[(num % 3)], + created_at: (new Date(2024 + (num % 6), (num % 12), (num % 30))), + updated_at: (new Date(2024 + (num % 6), (num % 12), ((num + 1) % 29))), + })); + + await t.context.collectionPgModel.insert(t.context.knex, collections); + await t.context.providerPgModel.insert(t.context.knex, providers); + await t.context.granulePgModel.insert(t.context.knex, granules); + await t.context.executionPgModel.insert(t.context.knex, executions); + await t.context.pdrPgModel.insert(t.context.knex, pdrs); + await t.context.reconciliationReportPgModel.insert(t.context.knex, reconReports); }); test.after.always(async (t) => { @@ -125,7 +106,7 @@ test.after.always(async (t) => { }); }); -test('StatsSearch returns correct response for basic granules query', async (t) => { +test('StatsSearch aggregate returns correct response for basic query with type granules', async (t) => { const { knex } = t.context; const AggregateSearch = new StatsSearch({}, 'granule'); const results = await AggregateSearch.aggregate(knex); @@ -139,7 +120,7 @@ test('StatsSearch returns correct response for basic granules query', async (t) t.deepEqual(results.count, expectedResponse); }); -test('StatsSearch filters correctly by date', async (t) => { +test('StatsSearch aggregate filters granules correctly by date', async (t) => { const { knex } = t.context; const queryStringParameters = { timestamp__from: `${(new Date(2020, 1, 28)).getTime()}`, @@ -158,7 +139,7 @@ test('StatsSearch filters correctly by date', async (t) => { t.deepEqual(results.count, expectedResponse); }); -test('StatsSearch filters executions correctly', async (t) => { +test('StatsSearch aggregate filters executions correctly', async (t) => { const { knex } = t.context; let queryStringParameters = { field: 'status', @@ -205,7 +186,7 @@ test('StatsSearch filters executions correctly', async (t) => { t.is(results3.meta.count, 1); }); -test('StatsSearch filters PDRs correctly', async (t) => { +test('StatsSearch aggregate filters PDRs correctly', async (t) => { const { knex } = t.context; let queryStringParameters = { field: 'status', @@ -247,7 +228,39 @@ test('StatsSearch filters PDRs correctly', async (t) => { t.deepEqual(results3.count, expectedResponse3); }); -test('StatsSearch returns correct response when queried by provider', async (t) => { +test('StatsSearch aggregate filters Reconciliation Reports correctly', async (t) => { + const { knex } = t.context; + let queryStringParameters = { + field: 'type', + }; + + const AggregateSearch = new StatsSearch({ queryStringParameters }, 'reconciliationReport'); + const results = await AggregateSearch.aggregate(knex); + const expectedResponse = [ + { key: 'Granule Inventory', count: 6 }, + { key: 'Granule Not Found', count: 6 }, + { key: 'Inventory', count: 6 }, + { key: 'ORCA Backup', count: 6 }, + ]; + t.is(results.meta.count, 24); + t.deepEqual(results.count, expectedResponse); + + queryStringParameters = { + field: 'status', + }; + + const AggregateSearch2 = new StatsSearch({ queryStringParameters }, 'reconciliationReport'); + const results2 = await AggregateSearch2.aggregate(knex); + const expectedResponse2 = [ + { key: 'Failed', count: 8 }, + { key: 'Generated', count: 8 }, + { key: 'Pending', count: 8 }, + ]; + t.is(results2.meta.count, 24); + t.deepEqual(results2.count, expectedResponse2); +}); + +test('StatsSearch returns correct aggregate response for type granule when queried by provider', async (t) => { const { knex } = t.context; const queryStringParameters = { field: 'status', @@ -261,7 +274,7 @@ test('StatsSearch returns correct response when queried by provider', async (t) t.deepEqual(results.count, expectedResponse); }); -test('StatsSearch returns correct response when queried by collection', async (t) => { +test('StatsSearch returns correct aggregate response for type granule when queried by collection', async (t) => { const { knex } = t.context; const queryStringParameters = { field: 'status', @@ -275,7 +288,7 @@ test('StatsSearch returns correct response when queried by collection', async (t t.deepEqual(results.count, expectedResponse); }); -test('StatsSearch returns correct response when queried by collection and provider', async (t) => { +test('StatsSearch returns correct aggregate response for type granule when queried by collection and provider', async (t) => { const { knex } = t.context; let queryStringParameters = { field: 'status', @@ -318,7 +331,7 @@ test('StatsSearch returns correct response when queried by collection and provid t.deepEqual(results3.count, expectedResponse3); }); -test('StatsSearch returns correct response when queried by error', async (t) => { +test('StatsSearch returns correct aggregate response for type granule when queried by error', async (t) => { const { knex } = t.context; let queryStringParameters = { field: 'error.Error.keyword', @@ -396,7 +409,7 @@ test('StatsSearch can query by infix and prefix when type is defined', async (t) t.deepEqual(results3.count, expectedResponse3); }); -test('StatsSummary works', async (t) => { +test('StatsSearch summary works', async (t) => { const { knex } = t.context; const StatsSummary = new StatsSearch({}, 'granule'); const results = await StatsSummary.summary(knex); diff --git a/packages/db/tests/search/test-field-mapping.js b/packages/db/tests/search/test-field-mapping.js index cccfccfde28..2d1af820556 100644 --- a/packages/db/tests/search/test-field-mapping.js +++ b/packages/db/tests/search/test-field-mapping.js @@ -239,3 +239,31 @@ test('mapQueryStringFieldToDbField correctly converts all rule api fields to db }, {}); t.deepEqual(dbQueryParams, expectedDbParameters); }); + +test('mapQueryStringFieldToDbField correctly converts all reconciliation report api fields to db fields', (t) => { + const queryStringParameters = { + name: 'some report name', + type: 'Granule Not Found', + status: 'Generated', + location: 's3://exampleBucket/examplePath', + createdAt: '1704100000000', + updatedAt: 1704100000000, + }; + + const expectedDbParameters = { + name: 'some report name', + type: 'Granule Not Found', + status: 'Generated', + location: 's3://exampleBucket/examplePath', + created_at: new Date(1704100000000), + updated_at: new Date(1704100000000), + }; + + const apiFieldsList = Object.entries(queryStringParameters) + .map(([name, value]) => ({ name, value })); + const dbQueryParams = apiFieldsList.reduce((acc, queryField) => { + const queryParam = mapQueryStringFieldToDbField('reconciliationReport', queryField); + return { ...acc, ...queryParam }; + }, {}); + t.deepEqual(dbQueryParams, expectedDbParameters); +}); diff --git a/packages/db/tests/search/test-queries.js b/packages/db/tests/search/test-queries.js index 0a1ecfff67e..9f3f7d1db27 100644 --- a/packages/db/tests/search/test-queries.js +++ b/packages/db/tests/search/test-queries.js @@ -77,6 +77,16 @@ test('convertQueryStringToDbQueryParameters correctly converts api query string t.deepEqual(dbQueryParams, expectedDbQueryParameters); }); +test('convertQueryStringToDbQueryParameters does not include limit/offset parameters if limit is explicitly set to null', (t) => { + const queryStringParameters = { + limit: null, + offset: 3, + }; + const dbQueryParams = convertQueryStringToDbQueryParameters('granule', queryStringParameters); + t.is(dbQueryParams.limit, undefined); + t.is(dbQueryParams.offset, undefined); +}); + test('convertQueryStringToDbQueryParameters correctly converts sortby error parameter to db query parameters', (t) => { const queryStringParameters = { sort_by: 'error.Error.keyword', diff --git a/packages/db/tests/translate/test-async-operations.js b/packages/db/tests/translate/test-async-operations.js index d4030b2f7f1..f824497cc72 100644 --- a/packages/db/tests/translate/test-async-operations.js +++ b/packages/db/tests/translate/test-async-operations.js @@ -14,7 +14,7 @@ test('translateApiAsyncOperationToPostgresAsyncOperation converts a camelCase re status: 'RUNNING', taskArn: 'aws:arn:ecs:task:someTask', description: 'dummy operation', - operationType: 'ES Index', + operationType: 'Reconciliation Report', }; const expected = { @@ -38,7 +38,7 @@ test('translateApiAsyncOperationToPostgresAsyncOperation parses output from JSON status: 'SUCCEEDED', taskArn: 'aws:arn:ecs:task:someTask', description: 'dummy operation', - operationType: 'ES Index', + operationType: 'Reconciliation Report', output: JSON.stringify(operationOutput), }; @@ -60,7 +60,7 @@ test('translateApiAsyncOperationToPostgresAsyncOperation parses output from JSON status: 'SUCCEEDED', taskArn: 'aws:arn:ecs:task:someTask', description: 'dummy operation', - operationType: 'ES Index', + operationType: 'Reconciliation Report', output: operationOutput, }; @@ -83,7 +83,7 @@ test('translateApiAsyncOperationToPostgresAsyncOperation parses output from stri status: 'SUCCEEDED', taskArn: 'aws:arn:ecs:task:someTask', description: 'dummy operation', - operationType: 'ES Index', + operationType: 'Reconciliation Report', output: operationOutput, }; @@ -106,7 +106,7 @@ test('translateApiAsyncOperationToPostgresAsyncOperation parses output from JSON status: 'SUCCEEDED', taskArn: 'aws:arn:ecs:task:someTask', description: 'dummy operation', - operationType: 'ES Index', + operationType: 'Reconciliation Report', output: operationOutput, }; @@ -127,7 +127,7 @@ test('translateApiAsyncOperationToPostgresAsyncOperation discards \'none\' outpu status: 'SUCCEEDED', taskArn: 'aws:arn:ecs:task:someTask', description: 'dummy operation', - operationType: 'ES Index', + operationType: 'Reconciliation Report', output: 'none', }; @@ -162,7 +162,7 @@ test('translatePostgresAsyncOperationToApiAsyncOperation translates PostgreSQL r status: 'RUNNING', taskArn, description, - operationType: 'ES Index', + operationType: 'Reconciliation Report', output: JSON.stringify({ test: 'output' }), createdAt: createdAt.getTime(), updatedAt: updatedAt.getTime(), diff --git a/packages/db/tests/translate/test-reconciliation-reports.js b/packages/db/tests/translate/test-reconciliation-reports.js new file mode 100644 index 00000000000..2314c78f700 --- /dev/null +++ b/packages/db/tests/translate/test-reconciliation-reports.js @@ -0,0 +1,59 @@ +const test = require('ava'); +const { randomId } = require('@cumulus/common/test-utils'); +const pick = require('lodash/pick'); + +const { translatePostgresReconReportToApiReconReport } = require('../../dist/translate/reconciliation_reports'); + +test('translatePostgresReconReportToApiReconReport translates a Postgres Reconciliation Report to an API Reconciliation Report', (t) => { + const createdTime = new Date(Date.now()); + const updatedTime = new Date(Date.now()); + + const pgReconReport = { + name: randomId('report'), + type: 'Granule Inventory', + status: 'Generated', + location: 's3://cumulus-test-sandbox-private/reconciliation-reports', + error: null, + created_at: createdTime, + updated_at: updatedTime, + }; + + const expectedApiReconReport = { + ...pick(pgReconReport, ['name', 'type', 'status', 'location']), + // no error b/c null or undefined should be removed + createdAt: createdTime.getTime(), + updatedAt: updatedTime.getTime(), + }; + + const translatedReport = translatePostgresReconReportToApiReconReport(pgReconReport); + + t.deepEqual(expectedApiReconReport, translatedReport); +}); + +test('translatePostgresReconReportToApiReconReport translates Postgres Reconciliation Report with an error to an API Reconciliation Report', (t) => { + const createdTime = new Date(Date.now()); + const updatedTime = new Date(Date.now()); + + const pgReconReport = { + name: randomId('report'), + type: 'Granule Not Found', + status: 'Failed', + location: 's3://cumulus-test-sandbox-private/reconciliation-reports', + error: { + Error: 'some error message', + Cause: 'some error cause', + }, + created_at: createdTime, + updated_at: updatedTime, + }; + + const expectedApiReconReport = { + ...pick(pgReconReport, ['name', 'type', 'status', 'location', 'error']), + createdAt: createdTime.getTime(), + updatedAt: updatedTime.getTime(), + }; + + const translatedReport = translatePostgresReconReportToApiReconReport(pgReconReport); + + t.deepEqual(expectedApiReconReport, translatedReport); +}); diff --git a/packages/db/tests/translate/test-rules.js b/packages/db/tests/translate/test-rules.js index b7804b97ae6..b7ddd5acec3 100644 --- a/packages/db/tests/translate/test-rules.js +++ b/packages/db/tests/translate/test-rules.js @@ -1,6 +1,7 @@ const test = require('ava'); const { translatePostgresRuleToApiRule, + translatePostgresRuleToApiRuleWithoutDbQuery, translateApiRuleToPostgresRule, translateApiRuleToPostgresRuleRaw, } = require('../../dist/translate/rules'); @@ -243,3 +244,62 @@ test('translatePostgresRuleToApiRule handles optional fields', async (t) => { expectedRule ); }); + +test('translatePostgresRuleToApiRuleWithoutDbQuery translates a postgres Rule', async (t) => { + const pgRecord = { + name: 'testRule', + workflow: 'testWorkflow', + type: 'onetime', + enabled: true, + collection_cumulus_id: 1, + provider_cumulus_id: 2, + execution_name_prefix: 'test', + value: 'abcd', + arn: 'arn:123', + log_event_arn: 'arn:987', + payload: { object: 'value' }, + meta: { + retries: 2, + visibility: 30, + more: 'meta', + }, + tags: ['tag1', 'tag2'], + queue_url: 'https://sqs.us-west-2.amazonaws.com/123456789012/my-queue', + created_at: new Date(), + updated_at: new Date(), + }; + + // Assume provider, collection are already retrieved, no query required + const fakeCollection = { name: 'abc', version: '123' }; + const fakeProvider = { name: 'abc' }; + + const expectedRule = { + name: pgRecord.name, + state: 'ENABLED', + workflow: pgRecord.workflow, + collection: fakeCollection, + provider: fakeProvider.name, + meta: pgRecord.meta, + payload: pgRecord.payload, + queueUrl: pgRecord.queue_url, + rule: { + type: pgRecord.type, + arn: pgRecord.arn, + logEventArn: pgRecord.log_event_arn, + value: pgRecord.value, + }, + executionNamePrefix: pgRecord.execution_name_prefix, + tags: pgRecord.tags, + createdAt: pgRecord.created_at.getTime(), + updatedAt: pgRecord.updated_at.getTime(), + }; + + t.deepEqual( + await translatePostgresRuleToApiRuleWithoutDbQuery( + pgRecord, + fakeCollection, + fakeProvider + ), + expectedRule + ); +}); diff --git a/packages/es-client/indexer.js b/packages/es-client/indexer.js index a6a770fa9b5..4fc40ec60d4 100644 --- a/packages/es-client/indexer.js +++ b/packages/es-client/indexer.js @@ -294,20 +294,6 @@ function indexReconciliationReport(esClient, payload, index = defaultIndexAlias, return genericRecordUpdate(esClient, payload.name, payload, index, type); } -/** - * Indexes the rule type on Elasticsearch - * - * @param {Object} esClient - Elasticsearch Connection object - * @param {Object} payload - the Rule record - * @param {string} index - Elasticsearch index alias (default defined in search.js) - * @param {string} type - Elasticsearch type (default: rule) - * @returns {Promise} Elasticsearch response - */ - -function indexRule(esClient, payload, index = defaultIndexAlias, type = 'rule') { - return genericRecordUpdate(esClient, payload.name, payload, index, type); -} - /** * Indexes the granule type on Elasticsearch * @@ -609,33 +595,6 @@ function deleteProvider({ }); } -/** - * Deletes the rule in Elasticsearch - * - * @param {Object} params - * @param {Object} params.esClient - Elasticsearch Connection object - * @param {string} params.name - the rule name - * @param {string[]} [params.ignore] - Array of response codes to ignore - * @param {string} params.index - Elasticsearch index alias (default defined in search.js) - * @param {string} params.type - Elasticsearch type (default: rule) - * @returns {Promise} Elasticsearch response - */ -function deleteRule({ - esClient, - name, - ignore, - index = defaultIndexAlias, - type = 'rule', -}) { - return deleteRecord({ - esClient, - id: name, - index, - type, - ignore, - }); -} - /** * Deletes the PDR in Elasticsearch * @@ -805,7 +764,6 @@ module.exports = { deleteProvider, deleteReconciliationReport, deleteRecord, - deleteRule, executionInvalidNullFields, granuleInvalidNullFields, genericRecordUpdate, @@ -816,7 +774,6 @@ module.exports = { indexPdr, indexProvider, indexReconciliationReport, - indexRule, updateAsyncOperation, upsertExecution, upsertGranule, diff --git a/packages/es-client/tests/test-es-indexer.js b/packages/es-client/tests/test-es-indexer.js index e3cf3a0f8f0..8b936b79863 100644 --- a/packages/es-client/tests/test-es-indexer.js +++ b/packages/es-client/tests/test-es-indexer.js @@ -168,29 +168,6 @@ test.serial('creating multiple deletedgranule records and retrieving them', asyn }); }); -test.serial('indexing a rule record', async (t) => { - const { esIndex, esClient } = t.context; - - const testRecord = { - name: randomString(), - }; - - const r = await indexer.indexRule(esClient, testRecord, esIndex); - - // make sure record is created - t.is(r.result, 'created'); - - // check the record exists - const record = await esClient.client.get({ - index: esIndex, - type: 'rule', - id: testRecord.name, - }).then((response) => response.body); - - t.is(record._id, testRecord.name); - t.is(typeof record._source.timestamp, 'number'); -}); - test.serial('indexing a provider record', async (t) => { const { esIndex, esClient } = t.context; @@ -617,31 +594,6 @@ test.serial('deleting a collection record', async (t) => { t.false(await esCollectionsClient.exists(collectionId)); }); -test.serial('deleting a rule record', async (t) => { - const { esIndex, esClient } = t.context; - const name = randomString(); - const testRecord = { - name, - }; - - await indexer.indexRule(esClient, testRecord, esIndex); - - // check the record exists - const esRulesClient = new Search( - {}, - 'rule', - esIndex - ); - t.true(await esRulesClient.exists(name)); - - await indexer.deleteRule({ - esClient, - name, - index: esIndex, - }); - t.false(await esRulesClient.exists(name)); -}); - test.serial('deleting a PDR record', async (t) => { const { esIndex, esClient } = t.context; diff --git a/packages/integration-tests/index.js b/packages/integration-tests/index.js index 55322ce2e56..60230e94f34 100644 --- a/packages/integration-tests/index.js +++ b/packages/integration-tests/index.js @@ -49,7 +49,7 @@ const lambdaStep = new LambdaStep(); /** * Wait for an AsyncOperation to reach a given status * - * Retries using exponental backoff until desired has been reached. If the + * Retries using exponential backoff until desired has been reached. If the * desired state is not reached an error is thrown. * * @param {Object} params - params diff --git a/packages/tf-inventory/src/inventory.js b/packages/tf-inventory/src/inventory.js index da596087d63..b1764938880 100644 --- a/packages/tf-inventory/src/inventory.js +++ b/packages/tf-inventory/src/inventory.js @@ -1,6 +1,6 @@ 'use strict'; -const { ecs, ec2, es } = require('@cumulus/aws-client/services'); +const { ecs, ec2 } = require('@cumulus/aws-client/services'); const mergeWith = require('lodash/mergeWith'); const difference = require('lodash/difference'); @@ -80,13 +80,9 @@ async function listAwsResources() { ec2Instances = [].concat(...ec2Instances.Reservations.map((e) => e.Instances)); ec2Instances = ec2Instances.map((inst) => inst.InstanceId); - let esDomainNames = await es().listDomainNames(); - esDomainNames = esDomainNames.DomainNames.map((e) => e.DomainName); - return { ecsClusters: ecsClusters.clusterArns, ec2Instances, - esDomainNames, }; } diff --git a/packages/tf-inventory/tests/inventory.js b/packages/tf-inventory/tests/inventory.js index 3ef54f23341..960406631c8 100644 --- a/packages/tf-inventory/tests/inventory.js +++ b/packages/tf-inventory/tests/inventory.js @@ -3,7 +3,7 @@ const test = require('ava'); const rewire = require('rewire'); const sinon = require('sinon'); -const { ecs, ec2, es } = require('@cumulus/aws-client/services'); +const { ecs, ec2 } = require('@cumulus/aws-client/services'); const inventory = rewire('../src/inventory'); const mergeResourceLists = inventory.__get__('mergeResourceLists'); const resourceDiff = inventory.__get__('resourceDiff'); @@ -15,7 +15,6 @@ let listResourcesForFileStub; let listTfStateFilesStub; let ecsStub; let ec2Stub; -let esStub; /** * @@ -28,7 +27,6 @@ function resourcesForStateFile(sf) { return { ecsClusters: ['clusterArn1', 'clusterArn2'], ec2Instances: ['i-000'], - esDomainNames: ['cumulus-1-es5vpc'], }; } @@ -36,7 +34,6 @@ function resourcesForStateFile(sf) { return { ecsClusters: ['clusterArn3'], ec2Instances: ['i-111', 'i-222'], - esDomainNames: ['cumulus-2-es5vpc'], }; } @@ -80,17 +77,6 @@ test.before(() => { ], }), }); - - esStub = sinon.stub(es(), 'listDomainNames') - .returns( - Promise.resolve({ - DomainNames: [ - { DomainName: 'cumulus-es5vpc' }, - { DomainName: 'cumulus-1-es5vpc' }, - { DomainName: 'cumulus-2-es5vpc' }, - ], - }) - ); }); test.after.always(() => { @@ -98,7 +84,6 @@ test.after.always(() => { listTfStateFilesStub.restore(); ecsStub.restore(); ec2Stub.restore(); - esStub.restore(); }); test('mergeResourceLists merges resource object by key', (t) => { @@ -235,7 +220,6 @@ test('listTfResources merges resources correctly', async (t) => { t.deepEqual(tfResources, { ecsClusters: ['clusterArn1', 'clusterArn2', 'clusterArn3'], ec2Instances: ['i-000', 'i-111', 'i-222'], - esDomainNames: ['cumulus-1-es5vpc', 'cumulus-2-es5vpc'], }); }); @@ -246,7 +230,6 @@ test('listAwsResources properly combines ec2 intsances', async (t) => { { ecsClusters: ['clusterArn1', 'clusterArn2', 'clusterArn3', 'clusterArn4'], ec2Instances: ['i-000', 'i-111', 'i-222', 'i-333'], - esDomainNames: ['cumulus-es5vpc', 'cumulus-1-es5vpc', 'cumulus-2-es5vpc'], }); }); @@ -257,6 +240,5 @@ test('reconcileResources returns only resources not specified in TF files', asyn { ecsClusters: ['clusterArn4'], ec2Instances: ['i-333'], - esDomainNames: ['cumulus-es5vpc'], }); }); diff --git a/packages/types/api/reconciliation_reports.d.ts b/packages/types/api/reconciliation_reports.d.ts new file mode 100644 index 00000000000..5473740551d --- /dev/null +++ b/packages/types/api/reconciliation_reports.d.ts @@ -0,0 +1,18 @@ +export type ReconciliationReportType = + 'Granule Inventory' | 'Granule Not Found' | 'Internal' | 'Inventory' | 'ORCA Backup'; +export type ReconciliationReportStatus = 'Generated' | 'Pending' | 'Failed'; + +export interface ApiReconciliationReport { + name: string, + type: ReconciliationReportType, + status: ReconciliationReportStatus, + location?: string, + error?: object, + createdAt?: number, + updatedAt?: number, +} + +export interface ApiReconciliationReportRecord extends ApiReconciliationReport { + createdAt: number, + updatedAt: number, +} diff --git a/tf-modules/archive/api.tf b/tf-modules/archive/api.tf index a1f12066b19..7cf1456366f 100644 --- a/tf-modules/archive/api.tf +++ b/tf-modules/archive/api.tf @@ -49,14 +49,10 @@ locals { EARTHDATA_CLIENT_PASSWORD = var.urs_client_password EcsCluster = var.ecs_cluster_name ENTITY_ID = var.saml_entity_id - ES_CONCURRENCY = var.es_request_concurrency - ES_HOST = var.elasticsearch_hostname - ES_INDEX_SHARDS = var.es_index_shards granule_sns_topic_arn = aws_sns_topic.report_granules_topic.arn execution_sns_topic_arn = aws_sns_topic.report_executions_topic.arn idleTimeoutMillis = var.rds_connection_timing_configuration.idleTimeoutMillis IDP_LOGIN = var.saml_idp_login - IndexFromDatabaseLambda = aws_lambda_function.index_from_database.arn invoke = var.schedule_sf_function_arn invokeArn = var.schedule_sf_function_arn invokeReconcileLambda = aws_lambda_function.create_reconciliation_report.arn diff --git a/tf-modules/archive/async_operation.tf b/tf-modules/archive/async_operation.tf index 5783a37e831..25985b01f1b 100644 --- a/tf-modules/archive/async_operation.tf +++ b/tf-modules/archive/async_operation.tf @@ -31,10 +31,6 @@ resource "aws_ecs_task_definition" "async_operation" { { "name": "databaseCredentialSecretArn", "value": "${var.rds_user_access_secret_arn}" - }, - { - "name": "ES_HOST", - "value": "${var.elasticsearch_hostname}" } ], "image": "${var.async_operation_image}", @@ -74,10 +70,6 @@ resource "aws_ecs_task_definition" "dead_letter_recovery_operation" { { "name": "databaseCredentialSecretArn", "value": "${var.rds_user_access_secret_arn}" - }, - { - "name": "ES_HOST", - "value": "${var.elasticsearch_hostname}" } ], "image": "${var.async_operation_image}", diff --git a/tf-modules/archive/bootstrap.tf b/tf-modules/archive/bootstrap.tf deleted file mode 100644 index 00468fe6d19..00000000000 --- a/tf-modules/archive/bootstrap.tf +++ /dev/null @@ -1,40 +0,0 @@ -resource "aws_lambda_function" "custom_bootstrap" { - function_name = "${var.prefix}-CustomBootstrap" - filename = "${path.module}/../../packages/api/dist/bootstrap/lambda.zip" - source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/bootstrap/lambda.zip") - handler = "index.handler" - role = var.lambda_processing_role_arn - runtime = "nodejs20.x" - timeout = lookup(var.lambda_timeouts, "CustomBootstrap", 300) - memory_size = lookup(var.lambda_memory_sizes, "CustomBootstrap", 512) - environment { - variables = { - stackName = var.prefix - system_bucket = var.system_bucket - ES_INDEX_SHARDS = var.es_index_shards - } - } - - tags = var.tags - - dynamic "vpc_config" { - for_each = length(var.lambda_subnet_ids) == 0 ? [] : [1] - content { - subnet_ids = var.lambda_subnet_ids - security_group_ids = local.lambda_security_group_ids - } - } -} - -data "aws_lambda_invocation" "custom_bootstrap" { - count = var.elasticsearch_hostname != null ? 1 : 0 - depends_on = [aws_lambda_function.custom_bootstrap] - function_name = aws_lambda_function.custom_bootstrap.function_name - - input = jsonencode( - { - elasticsearchHostname = var.elasticsearch_hostname - removeAliasConflict = var.elasticsearch_remove_index_alias_conflict - replacementTrigger = timestamp() - }) -} diff --git a/tf-modules/archive/bulk_operation.tf b/tf-modules/archive/bulk_operation.tf index 989c675b23b..17cc48889da 100644 --- a/tf-modules/archive/bulk_operation.tf +++ b/tf-modules/archive/bulk_operation.tf @@ -12,7 +12,6 @@ resource "aws_lambda_function" "bulk_operation" { acquireTimeoutMillis = var.rds_connection_timing_configuration.acquireTimeoutMillis createRetryIntervalMillis = var.rds_connection_timing_configuration.createRetryIntervalMillis createTimeoutMillis = var.rds_connection_timing_configuration.createTimeoutMillis - ES_HOST = var.elasticsearch_hostname granule_sns_topic_arn = aws_sns_topic.report_granules_topic.arn idleTimeoutMillis = var.rds_connection_timing_configuration.idleTimeoutMillis invoke = var.schedule_sf_function_arn diff --git a/tf-modules/archive/clean_executions.tf b/tf-modules/archive/clean_executions.tf index 6a55d920fe4..398c6658304 100644 --- a/tf-modules/archive/clean_executions.tf +++ b/tf-modules/archive/clean_executions.tf @@ -22,13 +22,9 @@ source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/clea environment { variables = { stackName = var.prefix - ES_HOST = var.elasticsearch_hostname - CLEANUP_RUNNING = var.cleanup_running - CLEANUP_NON_RUNNING = var.cleanup_non_running - - PAYLOAD_TIMEOUT = var.payload_timeout - - ES_INDEX = var.es_index + CLEANUP_RUNNING = var.cleanup_running + CLEANUP_NON_RUNNING = var.cleanup_non_running + PAYLOAD_TIMEOUT = var.payload_timeout UPDATE_LIMIT = var.update_limit } } diff --git a/tf-modules/archive/index_from_database.tf b/tf-modules/archive/index_from_database.tf deleted file mode 100644 index 61138dd4664..00000000000 --- a/tf-modules/archive/index_from_database.tf +++ /dev/null @@ -1,112 +0,0 @@ -resource "aws_lambda_function" "index_from_database" { - function_name = "${var.prefix}-IndexFromDatabase" - filename = "${path.module}/../../packages/api/dist/indexFromDatabase/lambda.zip" - source_code_hash = filebase64sha256("${path.module}/../../packages/api/dist/indexFromDatabase/lambda.zip") - handler = "index.handler" - role = aws_iam_role.index_from_database.arn - runtime = "nodejs20.x" - timeout = lookup(var.lambda_timeouts, "IndexFromDatabase", 300) - memory_size = lookup(var.lambda_memory_sizes, "IndexFromDatabase", 512) - environment { - variables = { - CMR_ENVIRONMENT = var.cmr_environment - CMR_HOST = var.cmr_custom_host - databaseCredentialSecretArn = var.rds_user_access_secret_arn - ES_CONCURRENCY = var.es_request_concurrency - ES_HOST = var.elasticsearch_hostname - ReconciliationReportsTable = var.dynamo_tables.reconciliation_reports.name - stackName = var.prefix - } - } - tags = var.tags - - dynamic "vpc_config" { - for_each = length(var.lambda_subnet_ids) == 0 ? [] : [1] - content { - subnet_ids = var.lambda_subnet_ids - security_group_ids = concat(local.lambda_security_group_ids, [var.rds_security_group]) - } - } -} - - -resource "aws_iam_role" "index_from_database" { - name = "${var.prefix}-index_from_database" - assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json - permissions_boundary = var.permissions_boundary_arn - - tags = var.tags -} - - -resource "aws_iam_role_policy" "index_from_database" { - name = "${var.prefix}_index_from_database_policy" - role = aws_iam_role.index_from_database.id - policy = data.aws_iam_policy_document.index_from_database.json -} - - -data "aws_iam_policy_document" "index_from_database" { - statement { - actions = ["ecs:RunTask"] - resources = [aws_ecs_task_definition.async_operation.arn] - } - - statement { - actions = [ - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "logs:DescribeLogStreams", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents", - ] - resources = ["*"] - } - - statement { - actions = [ - "dynamodb:GetItem", - "dynamodb:Scan", - ] - resources = [for k, v in var.dynamo_tables : v.arn] - } - - statement { - actions = ["dynamodb:Query"] - resources = [for k, v in var.dynamo_tables : "${v.arn}/index/*"] - } - - statement { - actions = [ - "dynamodb:GetRecords", - "dynamodb:GetShardIterator", - "dynamodb:DescribeStream", - "dynamodb:ListStreams" - ] - resources = [for k, v in var.dynamo_tables : "${v.arn}/stream/*"] - } - - statement { - actions = ["dynamodb:ListTables"] - resources = ["*"] - } - - statement { - actions = ["secretsmanager:GetSecretValue"] - resources = [ - aws_secretsmanager_secret.api_cmr_password.arn, - aws_secretsmanager_secret.api_launchpad_passphrase.arn, - var.rds_user_access_secret_arn - ] - } - - statement { - actions = [ - "ssm:GetParameter" - ] - resources = [aws_ssm_parameter.dynamo_table_names.arn] - } -} - diff --git a/tf-modules/archive/main.tf b/tf-modules/archive/main.tf index 890d48d84ba..e352a6c547b 100644 --- a/tf-modules/archive/main.tf +++ b/tf-modules/archive/main.tf @@ -10,7 +10,6 @@ terraform { locals { lambda_security_group_ids = compact([ aws_security_group.no_ingress_all_egress[0].id, - var.elasticsearch_security_group_id ]) all_bucket_names = [for k, v in var.buckets : v.name] all_non_internal_buckets = [for k, v in var.buckets : v.name if v.type != "internal"] diff --git a/tf-modules/archive/process_dead_letter_archive.tf b/tf-modules/archive/process_dead_letter_archive.tf index 4c6b198f655..c5b2448d459 100644 --- a/tf-modules/archive/process_dead_letter_archive.tf +++ b/tf-modules/archive/process_dead_letter_archive.tf @@ -22,7 +22,6 @@ resource "aws_lambda_function" "process_dead_letter_archive" { stackName = var.prefix system_bucket = var.system_bucket RDS_DEPLOYMENT_CUMULUS_VERSION = "9.0.0" - ES_HOST = var.elasticsearch_hostname } } diff --git a/tf-modules/archive/reconciliation_report.tf b/tf-modules/archive/reconciliation_report.tf index 36089d77949..cc04ae81cfe 100644 --- a/tf-modules/archive/reconciliation_report.tf +++ b/tf-modules/archive/reconciliation_report.tf @@ -14,10 +14,6 @@ resource "aws_lambda_function" "create_reconciliation_report" { CMR_ENVIRONMENT = var.cmr_environment CMR_HOST = var.cmr_custom_host DISTRIBUTION_ENDPOINT = var.distribution_url - ES_HOST = var.elasticsearch_hostname - ES_SCROLL = lookup(var.elasticsearch_client_config, "create_reconciliation_report_es_scroll_duration", "6m") - ES_SCROLL_SIZE = lookup(var.elasticsearch_client_config, "create_reconciliation_report_es_scroll_size", 1000) - ReconciliationReportsTable = var.dynamo_tables.reconciliation_reports.name stackName = var.prefix system_bucket = var.system_bucket cmr_client_id = var.cmr_client_id diff --git a/tf-modules/archive/sf_event_sqs_to_db_records.tf b/tf-modules/archive/sf_event_sqs_to_db_records.tf index 2ae35f465ff..e3cdb49b5aa 100644 --- a/tf-modules/archive/sf_event_sqs_to_db_records.tf +++ b/tf-modules/archive/sf_event_sqs_to_db_records.tf @@ -192,7 +192,6 @@ resource "aws_lambda_function" "sf_event_sqs_to_db_records" { pdr_sns_topic_arn = aws_sns_topic.report_pdrs_topic.arn RDS_DEPLOYMENT_CUMULUS_VERSION = "9.0.0" reapIntervalMillis = var.rds_connection_timing_configuration.reapIntervalMillis - ES_HOST = var.elasticsearch_hostname } } diff --git a/tf-modules/archive/start_async_operation_lambda.tf b/tf-modules/archive/start_async_operation_lambda.tf index 0957481916a..b05a7714358 100644 --- a/tf-modules/archive/start_async_operation_lambda.tf +++ b/tf-modules/archive/start_async_operation_lambda.tf @@ -15,7 +15,6 @@ resource "aws_lambda_function" "start_async_operation" { createTimeoutMillis = var.rds_connection_timing_configuration.createTimeoutMillis databaseCredentialSecretArn = var.rds_user_access_secret_arn EcsCluster = var.ecs_cluster_name - ES_HOST = var.elasticsearch_hostname idleTimeoutMillis = var.rds_connection_timing_configuration.idleTimeoutMillis reapIntervalMillis = var.rds_connection_timing_configuration.reapIntervalMillis stackName = var.prefix diff --git a/tf-modules/archive/variables.tf b/tf-modules/archive/variables.tf index 07af3668100..74ab989ec42 100644 --- a/tf-modules/archive/variables.tf +++ b/tf-modules/archive/variables.tf @@ -71,27 +71,6 @@ variable "ecs_task_role" { type = object({ name = string, arn = string}) } -variable "elasticsearch_domain_arn" { - type = string - default = null -} - -variable "elasticsearch_hostname" { - type = string - default = null -} - -variable "elasticsearch_security_group_id" { - type = string - default = "" -} - -variable "elasticsearch_remove_index_alias_conflict" { - type = bool - default = false - description = "Set to true to allow cumulus deployment bootstrap lambda to remove existing ES index named 'cumulus-alias' if it exists. Setting to false will cause deployment to fail on existing index" -} - variable "kinesis_inbound_event_logger_lambda_function_arn" { type = string } @@ -198,18 +177,6 @@ variable "cmr_search_client_config" { default = {} } -variable "elasticsearch_client_config" { - description = "Configuration parameters for Elasticsearch client for cumulus tasks" - type = map(string) - default = {} -} - -variable "es_request_concurrency" { - type = number - default = 10 - description = "Maximum number of concurrent requests to send to Elasticsearch. Used in index-from-database operation" -} - variable "lambda_memory_sizes" { description = "Configurable map of memory sizes for lambdas" type = map(number) @@ -358,12 +325,6 @@ variable "payload_timeout" { description = "Number of days to retain execution payload records in the database" } -variable "es_index" { - type = string - default = "cumulus" - description = "elasticsearch index to be affected" -} - variable "update_limit" { type = number default = 10000 @@ -376,12 +337,6 @@ variable "log_destination_arn" { description = "A shared AWS:Log:Destination that receives logs from log_groups" } -variable "es_index_shards" { - description = "The number of shards for the Elasticsearch index" - type = number - default = 2 -} - variable "cloudwatch_log_retention_periods" { type = map(number) description = "retention periods for the respective cloudwatch log group, these values will be used instead of default retention days" diff --git a/tf-modules/cumulus/README.md b/tf-modules/cumulus/README.md index a87d787c78a..94fd96de919 100644 --- a/tf-modules/cumulus/README.md +++ b/tf-modules/cumulus/README.md @@ -70,11 +70,5 @@ module "cumulus" { archive_api_users = ["urs-user1", "urs-user2"] sts_credentials_lambda_function_arn = "arn:aws:lambda:us-east-1:1234567890:function:sts-lambda" - - # Optional - elasticsearch_alarms = ["arn:aws:cloudwatch:us-east-1:12345:alarm:prefix-es-NodesLowAlarm"] - elasticsearch_domain_arn = "arn:aws:es:us-east-1:12345:domain/prefix-es" - elasticsearch_hostname = "prefix-es-abcdef.us-east-1.es.amazonaws.com" - elasticsearch_security_group_id = ["sg-12345"] } ``` diff --git a/tf-modules/cumulus/archive.tf b/tf-modules/cumulus/archive.tf index 84a4818efd0..a99b269d305 100644 --- a/tf-modules/cumulus/archive.tf +++ b/tf-modules/cumulus/archive.tf @@ -17,15 +17,6 @@ module "archive" { default_log_retention_days = var.default_log_retention_days cloudwatch_log_retention_periods = var.cloudwatch_log_retention_periods - elasticsearch_client_config = var.elasticsearch_client_config - elasticsearch_domain_arn = var.elasticsearch_domain_arn - elasticsearch_hostname = var.elasticsearch_hostname - elasticsearch_security_group_id = var.elasticsearch_security_group_id - elasticsearch_remove_index_alias_conflict = var.elasticsearch_remove_index_alias_conflict - - es_index_shards = var.es_index_shards - es_request_concurrency = var.es_request_concurrency - system_bucket = var.system_bucket buckets = var.buckets @@ -92,7 +83,6 @@ module "archive" { payload_timeout = var.payload_timeout - es_index = var.es_index update_limit = var.update_limit background_queue_url = module.ingest.background_queue_url diff --git a/tf-modules/cumulus/ecs_cluster.tf b/tf-modules/cumulus/ecs_cluster.tf index 438f0175bf7..0be8ea6c562 100644 --- a/tf-modules/cumulus/ecs_cluster.tf +++ b/tf-modules/cumulus/ecs_cluster.tf @@ -154,28 +154,6 @@ resource "aws_iam_role_policy" "ecs_cluster_instance" { policy = data.aws_iam_policy_document.ecs_cluster_instance_policy.json } -# Give ECS permission to access ES, if necessary -data "aws_iam_policy_document" "ecs_cluster_access_es_document" { - count = var.elasticsearch_domain_arn != null ? 1 : 0 - statement { - actions = [ - "es:ESHttpDelete", - "es:ESHttpGet", - "es:ESHttpHead", - "es:ESHttpPost", - "es:ESHttpPut" - ] - resources = [var.elasticsearch_domain_arn] - } -} - -resource "aws_iam_role_policy" "ecs_cluster_access_es_policy" { - name = "${var.prefix}_ecs_cluster_access_es_policy" - count = var.elasticsearch_domain_arn != null ? 1 : 0 - role = aws_iam_role.ecs_cluster_instance.id - policy = data.aws_iam_policy_document.ecs_cluster_access_es_document[0].json -} - resource "aws_iam_role_policy_attachment" "NGAPProtAppInstanceMinimalPolicy" { count = var.deploy_to_ngap ? 1 : 0 policy_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/NGAPProtAppInstanceMinimalPolicy" @@ -235,7 +213,6 @@ locals { security_group_ids = compact(concat( [ aws_security_group.ecs_cluster_instance.id, - var.elasticsearch_security_group_id, var.rds_security_group ], var.ecs_custom_sg_ids diff --git a/tf-modules/cumulus/migration_helper_async_operation.tf b/tf-modules/cumulus/migration_helper_async_operation.tf index 65955e311c9..ce898cfdaea 100644 --- a/tf-modules/cumulus/migration_helper_async_operation.tf +++ b/tf-modules/cumulus/migration_helper_async_operation.tf @@ -12,9 +12,6 @@ module "migration_helper_async_operation" { ecs_execution_role_arn = aws_iam_role.ecs_execution_role.arn ecs_task_role_arn = aws_iam_role.ecs_task_role.arn - elasticsearch_hostname = var.elasticsearch_hostname - elasticsearch_security_group_id = var.elasticsearch_security_group_id - lambda_subnet_ids = var.lambda_subnet_ids prefix = var.prefix diff --git a/tf-modules/cumulus/monitoring.tf b/tf-modules/cumulus/monitoring.tf index 61d40fcf1a2..fd112c944f1 100644 --- a/tf-modules/cumulus/monitoring.tf +++ b/tf-modules/cumulus/monitoring.tf @@ -2,7 +2,6 @@ module "monitoring" { source = "../monitoring" prefix = var.prefix - elasticsearch_alarms = var.elasticsearch_alarms ecs_service_alarms = var.ecs_service_alarms system_bucket = var.system_bucket } diff --git a/tf-modules/cumulus/reconciliation_report_migration.tf b/tf-modules/cumulus/reconciliation_report_migration.tf new file mode 100644 index 00000000000..c6132d136a4 --- /dev/null +++ b/tf-modules/cumulus/reconciliation_report_migration.tf @@ -0,0 +1,21 @@ +module "reconciliation_report_migration_lambda" { + source = "../../lambdas/reconciliation-report-migration" + + prefix = var.prefix + system_bucket = var.system_bucket + + dynamo_tables = var.dynamo_tables + + lambda_subnet_ids = var.lambda_subnet_ids + lambda_timeouts = var.lambda_timeouts + lambda_memory_sizes = var.lambda_memory_sizes + + permissions_boundary_arn = var.permissions_boundary_arn + + rds_security_group_id = var.rds_security_group + rds_user_access_secret_arn = var.rds_user_access_secret_arn + + tags = var.tags + vpc_id = var.vpc_id +} + diff --git a/tf-modules/cumulus/variables.tf b/tf-modules/cumulus/variables.tf index 0c3bb7a1113..24274b95633 100644 --- a/tf-modules/cumulus/variables.tf +++ b/tf-modules/cumulus/variables.tf @@ -82,30 +82,6 @@ variable "ecs_cluster_min_size" { type = number } -variable "elasticsearch_remove_index_alias_conflict" { - type = bool - default = false - description = "Set to true to allow cumulus deployment bootstrap lambda to remove existing ES index named 'cumulus-alias' if it exists. Setting to false will cause deployment to fail on existing index" -} - -variable "elasticsearch_domain_arn" { - description = "The ARN of an Elasticsearch domain to use for storing data" - type = string - default = null -} - -variable "elasticsearch_hostname" { - description = "The hostname of an Elasticsearch domain to use for storing data" - type = string - default = null -} - -variable "elasticsearch_security_group_id" { - description = "The ID of the security group for the Elasticsearch domain specified by `elasticsearch_domain_arn`" - type = string - default = "" -} - variable "lambda_memory_sizes" { description = "Configurable map of memory sizes for lambdas" type = map(number) @@ -181,12 +157,6 @@ variable "cmr_search_client_config" { default = {} } -variable "elasticsearch_client_config" { - description = "Configuration parameters for Elasticsearch client" - type = map(string) - default = {} -} - variable "archive_api_port" { description = "Port number that should be used for archive API requests" type = number @@ -321,18 +291,6 @@ variable "ecs_service_alarms" { default = [] } -variable "elasticsearch_alarms" { - description = "List of Cloudwatch alarms monitoring Elasticsearch domain" - type = list(object({ name = string, arn = string })) - default = [] -} - -variable "es_request_concurrency" { - type = number - default = 10 - description = "Maximum number of concurrent requests to send to Elasticsearch. Used in index-from-database operation" -} - variable "key_name" { description = "Name of EC2 key pair for accessing EC2 instances" type = string @@ -568,12 +526,6 @@ variable "payload_timeout" { description = "Number of days to retain execution payload records in the database" } -variable "es_index" { - type = string - default = "cumulus" - description = "elasticsearch index to be affected" -} - variable "update_limit" { type = number default = 10000 @@ -592,12 +544,6 @@ variable "additional_log_groups_to_elk" { default = {} } -variable "es_index_shards" { - description = "The number of shards for the Elasticsearch index" - type = number - default = 2 -} - variable "ecs_custom_sg_ids" { description = "User defined security groups to add to the Core ECS cluster" type = list(string) diff --git a/tf-modules/monitoring/.terraform.tfvars.sample b/tf-modules/monitoring/.terraform.tfvars.sample index d8bd52aac6b..a500bc7c179 100644 --- a/tf-modules/monitoring/.terraform.tfvars.sample +++ b/tf-modules/monitoring/.terraform.tfvars.sample @@ -1,17 +1,6 @@ # Required prefix = "myprefix" -elasticsearch_alarms = [ - { - "arn" = "es-alarm1-arn" - "name" = "es-alarm1" - }, - { - "arn" = "alarm2-arn" - "name" = "alarm2" - } -] - ecs_service_alarms = [ { "arn" = "ecs-alarm1-arn" diff --git a/tf-modules/monitoring/cloudwatch-dashboard.tf b/tf-modules/monitoring/cloudwatch-dashboard.tf index e19fe409080..c8fe7d7637b 100644 --- a/tf-modules/monitoring/cloudwatch-dashboard.tf +++ b/tf-modules/monitoring/cloudwatch-dashboard.tf @@ -12,38 +12,6 @@ resource "aws_cloudwatch_dashboard" "cloudwatch_dashboard" { dashboard_body = <