diff --git a/README.md b/README.md index c962bc6a..30523a7a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ - [Architecture](#architecture) - [Migration](#migration) - [0.4.x -\> 0.5.x](#04x---05x) - - [OpenSearch migration](#opensearch-migration) + - [Elasticsearch to OpenSearch Migration](#elasticsearch-to-opensearch-migration) - [Preferred Elasticsearch to OpenSearch Migration Process](#preferred-elasticsearch-to-opensearch-migration-process) - [0.3.x -\> 0.4.x](#03x---04x) - [Elasticsearch upgrade from 7.9 to 7.10](#elasticsearch-upgrade-from-79-to-710) @@ -119,7 +119,7 @@ apiLambda --> opensearch ### 0.4.x -> 0.5.x -#### OpenSearch migration +#### Elasticsearch to OpenSearch Migration By default, a new deployment of 0.5.x will use OpenSearch instead of Elasticsearch. There are three options if you have an existing deployment that uses Elasticsearch: @@ -142,6 +142,9 @@ are three options if you have an existing deployment that uses Elasticsearch: upgrade the Elasticsearch domain to OpenSearch, and connect the domain to the new CF Stack. This is described below. +Additionally, the `ES_HOST` variable used in the serverless.yml file has been +renamed `OPENSEARCH_HOST`. + #### Preferred Elasticsearch to OpenSearch Migration Process **Note! The migration must be done carefully to avoid losing the database!** diff --git a/src/lambdas/api/app.js b/src/lambdas/api/app.js index a2f7fd60..3ec97c07 100644 --- a/src/lambdas/api/app.js +++ b/src/lambdas/api/app.js @@ -5,7 +5,7 @@ const createError = require('http-errors') const express = require('express') const logger = require('morgan') const path = require('path') -const es = require('../../lib/es') +const database = require('../../lib/database') const api = require('../../lib/api') const { readFile } = require('../../lib/fs') const { addEndpoint } = require('./middleware/add-endpoint') @@ -28,7 +28,7 @@ app.use(addEndpoint) app.get('/', async (req, res, next) => { try { - res.json(await api.getCatalog(txnEnabled, es, req.endpoint)) + res.json(await api.getCatalog(txnEnabled, database, req.endpoint)) } catch (error) { next(error) } @@ -63,7 +63,7 @@ app.get('/conformance', async (_req, res, next) => { app.get('/search', async (req, res, next) => { try { res.type('application/geo+json') - res.json(await api.searchItems(null, req.query, es, req.endpoint, 'GET')) + res.json(await api.searchItems(null, req.query, database, req.endpoint, 'GET')) } catch (error) { if (error instanceof api.ValidationError) { next(createError(400, error.message)) @@ -76,7 +76,7 @@ app.get('/search', async (req, res, next) => { app.post('/search', async (req, res, next) => { try { res.type('application/geo+json') - res.json(await api.searchItems(null, req.body, es, req.endpoint, 'POST')) + res.json(await api.searchItems(null, req.body, database, req.endpoint, 'POST')) } catch (error) { if (error instanceof api.ValidationError) { next(createError(400, error.message)) @@ -88,7 +88,7 @@ app.post('/search', async (req, res, next) => { app.get('/aggregate', async (req, res, next) => { try { - res.json(await api.aggregate(req.query, es, req.endpoint, 'GET')) + res.json(await api.aggregate(req.query, database, req.endpoint, 'GET')) } catch (error) { if (error instanceof api.ValidationError) { next(createError(400, error.message)) @@ -100,7 +100,7 @@ app.get('/aggregate', async (req, res, next) => { app.post('/aggregate', async (req, res, next) => { try { - res.json(await api.aggregate(req.body, es, req.endpoint, 'POST')) + res.json(await api.aggregate(req.body, database, req.endpoint, 'POST')) } catch (error) { if (error instanceof api.ValidationError) { next(createError(400, error.message)) @@ -112,7 +112,7 @@ app.post('/aggregate', async (req, res, next) => { app.get('/collections', async (req, res, next) => { try { - res.json(await api.getCollections(es, req.endpoint)) + res.json(await api.getCollections(database, req.endpoint)) } catch (error) { next(error) } @@ -122,7 +122,7 @@ app.post('/collections', async (req, res, next) => { if (txnEnabled) { const collectionId = req.body.collection try { - await api.createCollection(req.body, es) + await api.createCollection(req.body, database) res.location(`${req.endpoint}/collections/${collectionId}`) res.sendStatus(201) } catch (error) { @@ -142,7 +142,7 @@ app.post('/collections', async (req, res, next) => { app.get('/collections/:collectionId', async (req, res, next) => { const { collectionId } = req.params try { - const response = await api.getCollection(collectionId, es, req.endpoint) + const response = await api.getCollection(collectionId, database, req.endpoint) if (response instanceof Error) next(createError(404)) else res.json(response) @@ -154,14 +154,14 @@ app.get('/collections/:collectionId', async (req, res, next) => { app.get('/collections/:collectionId/items', async (req, res, next) => { const { collectionId } = req.params try { - const response = await api.getCollection(collectionId, es, req.endpoint) + const response = await api.getCollection(collectionId, database, req.endpoint) if (response instanceof Error) next(createError(404)) else { const items = await api.searchItems( collectionId, req.query, - es, + database, req.endpoint, 'GET' ) @@ -185,11 +185,11 @@ app.post('/collections/:collectionId/items', async (req, res, next) => { if (req.body.collection && req.body.collection !== collectionId) { next(createError(400, 'Collection resource URI must match collection in body')) } else { - const collectionRes = await api.getCollection(collectionId, es, req.endpoint) + const collectionRes = await api.getCollection(collectionId, database, req.endpoint) if (collectionRes instanceof Error) next(createError(404)) try { req.body.collection = collectionId - await api.createItem(req.body, es) + await api.createItem(req.body, database) res.location(`${req.endpoint}/collections/${collectionId}/items/${itemId}`) res.sendStatus(201) } catch (error) { @@ -214,7 +214,7 @@ app.get('/collections/:collectionId/items/:itemId', async (req, res, next) => { const response = await api.getItem( collectionId, itemId, - es, + database, req.endpoint ) @@ -242,13 +242,13 @@ app.put('/collections/:collectionId/items/:itemId', async (req, res, next) => { } else if (req.body.id && req.body.id !== itemId) { next(createError(400, 'Item ID in resource URI must match id in body')) } else { - const itemRes = await api.getItem(collectionId, itemId, es, req.endpoint) + const itemRes = await api.getItem(collectionId, itemId, database, req.endpoint) if (itemRes instanceof Error) next(createError(404)) else { req.body.collection = collectionId req.body.id = itemId try { - await api.updateItem(req.body, es) + await api.updateItem(req.body, database) res.sendStatus(204) } catch (error) { if (error instanceof Error @@ -275,13 +275,13 @@ app.patch('/collections/:collectionId/items/:itemId', async (req, res, next) => } else if (req.body.id && req.body.id !== itemId) { next(createError(400, 'Item ID in resource URI must match id in body')) } else { - const itemRes = await api.getItem(collectionId, itemId, es, req.endpoint) + const itemRes = await api.getItem(collectionId, itemId, database, req.endpoint) if (itemRes instanceof Error) next(createError(404)) else { try { //const item = await api.partialUpdateItem( - collectionId, itemId, req.body, es, req.endpoint + collectionId, itemId, req.body, database, req.endpoint ) // res.type('application/geo+json') // res.json(item) @@ -300,7 +300,7 @@ app.delete('/collections/:collectionId/items/:itemId', async (req, res, next) => if (txnEnabled) { const { collectionId, itemId } = req.params try { - const response = await api.deleteItem(collectionId, itemId, es) + const response = await api.deleteItem(collectionId, itemId, database) if (response instanceof Error) next(createError(500)) else { res.sendStatus(204) diff --git a/src/lambdas/ingest/index.js b/src/lambdas/ingest/index.js index 35f35dd4..5e0af660 100644 --- a/src/lambdas/ingest/index.js +++ b/src/lambdas/ingest/index.js @@ -1,6 +1,6 @@ const { default: got } = require('got') -const esClient = require('../../lib/esClient.js') -const stream = require('../../lib/esStream.js') +const dbClient = require('../../lib/databaseClient.js') +const stream = require('../../lib/databaseStream.js') const ingest = require('../../lib/ingest.js') const s3Utils = require('../../lib/s3-utils') @@ -53,7 +53,7 @@ module.exports.handler = async function handler(event, context) { logger.debug(`Event: ${JSON.stringify(event, undefined, 2)}`) if (event.create_indices) { - await esClient.createIndex('collections') + await dbClient.createIndex('collections') } const stacItems = isSqsEvent(event) diff --git a/src/lib/api.js b/src/lib/api.js index 974f82b0..d4172c71 100644 --- a/src/lib/api.js +++ b/src/lib/api.js @@ -1,7 +1,7 @@ const { pickBy, assign, get: getNested } = require('lodash') const extent = require('@mapbox/extent') const { DateTime } = require('luxon') -const { isIndexNotFoundError } = require('./es') +const { isIndexNotFoundError } = require('./database') const logger = console // max number of collections to retrieve diff --git a/src/lib/es.js b/src/lib/database.js similarity index 94% rename from src/lib/es.js rename to src/lib/database.js index b1b86bc2..b1031fb1 100644 --- a/src/lib/es.js +++ b/src/lib/database.js @@ -1,4 +1,4 @@ -const esClient = require('./esClient') +const dbClient = require('./databaseClient') const logger = console //require('./logger') const COLLECTIONS_INDEX = process.env.COLLECTIONS_INDEX || 'collections' @@ -228,11 +228,11 @@ function buildFieldsFilter(parameters) { * */ async function indexCollection(collection) { - const client = await esClient.client() + const client = await dbClient.client() const exists = await client.indices.exists({ index: COLLECTIONS_INDEX }) if (!exists.body) { - await esClient.createIndex(COLLECTIONS_INDEX) + await dbClient.createIndex(COLLECTIONS_INDEX) } const collectionDocResponse = await client.index({ @@ -242,7 +242,7 @@ async function indexCollection(collection) { opType: 'create' }) - const indexCreateResponse = await esClient.createIndex(collection.id) + const indexCreateResponse = await dbClient.createIndex(collection.id) return [collectionDocResponse, indexCreateResponse] } @@ -252,7 +252,7 @@ async function indexCollection(collection) { * */ async function indexItem(item) { - const client = await esClient.client() + const client = await dbClient.client() const exists = await client.indices.exists({ index: item.collection }) if (!exists.body) { @@ -282,7 +282,7 @@ async function indexItem(item) { * */ async function partialUpdateItem(collectionId, itemId, updateFields) { - const client = await esClient.client() + const client = await dbClient.client() // Handle inserting required default properties to `updateFields` const requiredProperties = { @@ -309,7 +309,7 @@ async function partialUpdateItem(collectionId, itemId, updateFields) { } async function deleteItem(collectionId, itemId) { - const client = await esClient.client() + const client = await dbClient.client() if (client === undefined) throw new Error('Client is undefined') return await client.delete_by_query({ index: collectionId, @@ -318,9 +318,9 @@ async function deleteItem(collectionId, itemId) { }) } -async function esQuery(parameters) { +async function dbQuery(parameters) { logger.info(`Search database query: ${JSON.stringify(parameters)}`) - const client = await esClient.client() + const client = await dbClient.client() if (client === undefined) throw new Error('Client is undefined') const response = await client.search(parameters) logger.info(`Response: ${JSON.stringify(response)}`) @@ -329,7 +329,7 @@ async function esQuery(parameters) { // get single collection async function getCollection(collectionId) { - const response = await esQuery({ + const response = await dbQuery({ index: COLLECTIONS_INDEX, body: buildIdQuery(collectionId) }) @@ -342,7 +342,7 @@ async function getCollection(collectionId) { // get all collections async function getCollections(page = 1, limit = 100) { try { - const response = await esQuery({ + const response = await dbQuery({ index: COLLECTIONS_INDEX, size: limit, from: (page - 1) * limit @@ -391,18 +391,18 @@ async function constructSearchParams(parameters, page, limit) { async function search(parameters, page, limit = 10) { const searchParams = await constructSearchParams(parameters, page, limit) - const esResponse = await esQuery({ + const dbResponse = await dbQuery({ ignore_unavailable: true, allow_no_indices: true, ...searchParams }) - const results = esResponse.body.hits.hits.map((r) => (r._source)) + const results = dbResponse.body.hits.hits.map((r) => (r._source)) const response = { results, context: { limit: Number(limit), - matched: esResponse.body.hits.total.value, + matched: dbResponse.body.hits.total.value, returned: results.length } } @@ -475,13 +475,13 @@ async function aggregate(parameters) { } } - const esResponse = await esQuery({ + const dbResponse = await dbQuery({ ignore_unavailable: true, allow_no_indices: true, ...searchParams }) - return esResponse + return dbResponse } const getItem = async (collectionId, itemId) => { @@ -505,7 +505,7 @@ const getItemCreated = async (collectionId, itemId) => { * */ async function updateItem(item) { - const client = await esClient.client() + const client = await dbClient.client() const exists = await client.indices.exists({ index: item.collection }) if (!exists.body) { diff --git a/src/lib/esClient.js b/src/lib/databaseClient.js similarity index 95% rename from src/lib/esClient.js rename to src/lib/databaseClient.js index d24665a6..f6407be8 100644 --- a/src/lib/esClient.js +++ b/src/lib/databaseClient.js @@ -12,7 +12,7 @@ const logger = console //require('./logger') const { collectionsIndexConfiguration } = require('../../fixtures/collections') const { itemsIndexConfiguration } = require('../../fixtures/items') -let _esClient +let _dbClient function createClientWithUsernameAndPassword(host, username, password) { const protocolAndHost = host.split('://') @@ -71,19 +71,19 @@ async function connect() { } // get existing search database client or create a new one -async function esClient() { - if (_esClient) { +async function dbClient() { + if (_dbClient) { logger.debug('Using existing search database connection') } else { - _esClient = await connect() + _dbClient = await connect() logger.debug('Connected to search database') } - return _esClient + return _dbClient } async function createIndex(index) { - const client = await esClient() + const client = await dbClient() const exists = await client.indices.exists({ index }) const indexConfiguration = index === 'collections' ? collectionsIndexConfiguration() : itemsIndexConfiguration() @@ -101,7 +101,7 @@ async function createIndex(index) { } module.exports = { - client: esClient, + client: dbClient, createIndex, connect } diff --git a/src/lib/esStream.js b/src/lib/databaseStream.js similarity index 86% rename from src/lib/esStream.js rename to src/lib/databaseStream.js index 2680c9a1..4b07becd 100644 --- a/src/lib/esStream.js +++ b/src/lib/databaseStream.js @@ -1,8 +1,8 @@ const _stream = require('stream') const through2 = require('through2') const logger = console //require('./logger') -const esClient = require('./esClient') -const { getItemCreated } = require('./es') +const dbClient = require('./databaseClient') +const { getItemCreated } = require('./database') const COLLECTIONS_INDEX = process.env.COLLECTIONS_INDEX || 'collections' @@ -66,7 +66,7 @@ class SearchDatabaseWritableStream extends _stream.Writable { // if this was a collection, then add a new index with collection name if (index === COLLECTIONS_INDEX) { - await esClient.createIndex(id) + await dbClient.createIndex(id) } return next() @@ -98,11 +98,11 @@ class SearchDatabaseWritableStream extends _stream.Writable { // Given an input stream and a transform, write records to a search database instance async function stream() { - let esStreams + let dbStreams try { - const client = await esClient.client() + const client = await dbClient.client() - const toEs = through2.obj({ objectMode: true }, async (data, encoding, next) => { + const toDB = through2.obj({ objectMode: true }, async (data, encoding, next) => { let index = '' logger.debug(`Data: ${JSON.stringify(data)}`) if (data && data.hasOwnProperty('extent')) { @@ -117,38 +117,38 @@ async function stream() { // remove any hierarchy links in a non-mutating way const hlinks = ['self', 'root', 'parent', 'child', 'collection', 'item', 'items'] const links = data.links.filter((link) => !hlinks.includes(link.rel)) - const esDataObject = { ...data, links } + const dbDataObject = { ...data, links } if (data.hasOwnProperty('properties')) { const now = (new Date()).toISOString() const created = (await getItemCreated(data.collection, data.id)) || now - esDataObject.properties.created = created - esDataObject.properties.updated = now + dbDataObject.properties.created = created + dbDataObject.properties.updated = now } // create ES record const record = { index, - id: esDataObject.id, + id: dbDataObject.id, action: 'index', _retry_on_conflict: 3, - body: esDataObject + body: dbDataObject } next(null, record) }) - const esStream = new SearchDatabaseWritableStream({ client: client }, { + const dbStream = new SearchDatabaseWritableStream({ client: client }, { objectMode: true, highWaterMark: Number(process.env.INGEST_BATCH_SIZE || process.env.ES_BATCH_SIZE) || 500 }) - esStreams = { toEs, esStream } + dbStreams = { toDB, dbStream } } catch (error) { logger.error(error) } - return esStreams + return dbStreams } module.exports = stream diff --git a/src/lib/index.js b/src/lib/index.js index bb680b23..c36aa917 100644 --- a/src/lib/index.js +++ b/src/lib/index.js @@ -3,6 +3,6 @@ module.exports = { api: require('./api'), es: require('./es'), - esStream: require('./esStream'), + dbStream: require('./databaseStream'), ingest: require('./ingest') } diff --git a/src/lib/ingest.js b/src/lib/ingest.js index 7b763f90..dbe87838 100644 --- a/src/lib/ingest.js +++ b/src/lib/ingest.js @@ -4,12 +4,12 @@ const logger = console //require('./logger') async function ingestItem(item, stream) { const readable = new Readable({ objectMode: true }) - const { toEs, esStream } = await stream() + const { toDB, dbStream } = await stream() const promise = new Promise((resolve, reject) => { pump( readable, - toEs, - esStream, + toDB, + dbStream, (error) => { if (error) { logger.info(error) @@ -28,12 +28,12 @@ async function ingestItem(item, stream) { async function ingestItems(items, stream) { const readable = new Readable({ objectMode: true }) - const { toEs, esStream } = await stream() + const { toDB, dbStream } = await stream() const promise = new Promise((resolve, reject) => { pump( readable, - toEs, - esStream, + toDB, + dbStream, (error) => { if (error) { logger.info(error) diff --git a/tests/helpers/es.js b/tests/helpers/database.js similarity index 90% rename from tests/helpers/es.js rename to tests/helpers/database.js index 6e6dc340..3a66ba8d 100644 --- a/tests/helpers/es.js +++ b/tests/helpers/database.js @@ -1,6 +1,6 @@ // @ts-check -const { connect, createIndex } = require('../../src/lib/esClient') +const { connect, createIndex } = require('../../src/lib/databaseClient') /** * @returns {Promise} diff --git a/tests/helpers/ingest.js b/tests/helpers/ingest.js index bd4e8850..54c55d2c 100644 --- a/tests/helpers/ingest.js +++ b/tests/helpers/ingest.js @@ -4,7 +4,7 @@ const awsClients = require('../../src/lib/aws-clients') const { handler } = require('../../src/lambdas/ingest') const { sqsTriggerLambda } = require('./sqs') const { nullLoggerContext } = require('./context') -const { refreshIndices } = require('./es') +const { refreshIndices } = require('./database') const { loadFixture } = require('./utils') /** diff --git a/tests/helpers/system-tests.js b/tests/helpers/system-tests.js index 4af5afd4..65bcf5aa 100644 --- a/tests/helpers/system-tests.js +++ b/tests/helpers/system-tests.js @@ -5,7 +5,7 @@ const { promisify } = require('util') const fs = require('fs') const path = require('path') const { startApi } = require('./api') -const { createCollectionsIndex, refreshIndices } = require('./es') +const { createCollectionsIndex, refreshIndices } = require('./database') const { createTopic, addSnsToSqsSubscription } = require('./sns') const { createQueue, getQueueArn } = require('./sqs') diff --git a/tests/system/test-api-collection-items-get.js b/tests/system/test-api-collection-items-get.js index 5f559be6..c915052d 100644 --- a/tests/system/test-api-collection-items-get.js +++ b/tests/system/test-api-collection-items-get.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-collection-post.js b/tests/system/test-api-collection-post.js index 401215fe..456b7127 100644 --- a/tests/system/test-api-collection-post.js +++ b/tests/system/test-api-collection-post.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { randomId } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-get-collection.js b/tests/system/test-api-get-collection.js index 9c05cdee..d280ea32 100644 --- a/tests/system/test-api-get-collection.js +++ b/tests/system/test-api-get-collection.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-get-collections.js b/tests/system/test-api-get-collections.js index c406dbe5..324aad55 100644 --- a/tests/system/test-api-get-collections.js +++ b/tests/system/test-api-get-collections.js @@ -1,7 +1,7 @@ const test = require('ava') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') -const { refreshIndices, deleteAllIndices } = require('../helpers/es') +const { refreshIndices, deleteAllIndices } = require('../helpers/database') const systemTests = require('../helpers/system-tests') test.before(async (t) => { diff --git a/tests/system/test-api-get-root.js b/tests/system/test-api-get-root.js index 3708c73e..0fe2f3c4 100644 --- a/tests/system/test-api-get-root.js +++ b/tests/system/test-api-get-root.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { randomId } = require('../helpers/utils') const { startApi } = require('../helpers/api') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-item-delete.js b/tests/system/test-api-item-delete.js index 41a94086..a4142fd3 100644 --- a/tests/system/test-api-item-delete.js +++ b/tests/system/test-api-item-delete.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-item-get.js b/tests/system/test-api-item-get.js index 90389293..e97a3a87 100644 --- a/tests/system/test-api-item-get.js +++ b/tests/system/test-api-item-get.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-item-patch.js b/tests/system/test-api-item-patch.js index d7d1adef..f29b5687 100644 --- a/tests/system/test-api-item-patch.js +++ b/tests/system/test-api-item-patch.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-item-post.js b/tests/system/test-api-item-post.js index 05e6db4c..e9560624 100644 --- a/tests/system/test-api-item-post.js +++ b/tests/system/test-api-item-post.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-item-put.js b/tests/system/test-api-item-put.js index 2352b765..9f264e6a 100644 --- a/tests/system/test-api-item-put.js +++ b/tests/system/test-api-item-put.js @@ -1,5 +1,5 @@ const test = require('ava') -const { deleteAllIndices } = require('../helpers/es') +const { deleteAllIndices } = require('../helpers/database') const { ingestItem } = require('../helpers/ingest') const { randomId, loadFixture } = require('../helpers/utils') const systemTests = require('../helpers/system-tests') diff --git a/tests/system/test-api-search-get.js b/tests/system/test-api-search-get.js index ba5d7f7c..76bdaf5b 100644 --- a/tests/system/test-api-search-get.js +++ b/tests/system/test-api-search-get.js @@ -1,9 +1,9 @@ const test = require('ava') const { default: got } = require('got') -const { deleteAllIndices, refreshIndices } = require('../helpers/es') +const { deleteAllIndices, refreshIndices } = require('../helpers/database') const { randomId } = require('../helpers/utils') const ingest = require('../../src/lib/ingest') -const stream = require('../../src/lib/esStream') +const stream = require('../../src/lib/databaseStream') const systemTests = require('../helpers/system-tests') test.before(async (t) => { diff --git a/tests/system/test-api-search-post.js b/tests/system/test-api-search-post.js index cfb14d29..04f9ff4a 100644 --- a/tests/system/test-api-search-post.js +++ b/tests/system/test-api-search-post.js @@ -1,10 +1,10 @@ const test = require('ava') -const { deleteAllIndices, refreshIndices } = require('../helpers/es') +const { deleteAllIndices, refreshIndices } = require('../helpers/database') const { randomId } = require('../helpers/utils') const ingest = require('../../src/lib/ingest') const intersectsGeometry = require('../fixtures/stac/intersectsGeometry.json') -const stream = require('../../src/lib/esStream') +const stream = require('../../src/lib/databaseStream') const systemTests = require('../helpers/system-tests') const ingestEntities = async (fixtures) => { diff --git a/tests/system/test-ingest.js b/tests/system/test-ingest.js index 532cc126..850971e0 100644 --- a/tests/system/test-ingest.js +++ b/tests/system/test-ingest.js @@ -5,7 +5,7 @@ const { getCollectionIds, getItem } = require('../helpers/api') const { handler } = require('../../src/lambdas/ingest') const { loadFixture, randomId } = require('../helpers/utils') const { nullLoggerContext } = require('../helpers/context') -const { refreshIndices, deleteAllIndices } = require('../helpers/es') +const { refreshIndices, deleteAllIndices } = require('../helpers/database') const { sqsTriggerLambda, purgeQueue } = require('../helpers/sqs') const awsClients = require('../../src/lib/aws-clients') const systemTests = require('../helpers/system-tests') diff --git a/tests/unit/test-es.js b/tests/unit/test-es.js index 3047c5a1..24aa575f 100644 --- a/tests/unit/test-es.js +++ b/tests/unit/test-es.js @@ -1,5 +1,5 @@ const test = require('ava') -const es = require('../../src/lib/es') +const db = require('../../src/lib/database') test('search id parameter doesnt override other parameters', async (t) => { const ids = 'a,b,c' @@ -8,7 +8,7 @@ test('search id parameter doesnt override other parameters', async (t) => { ids: ids, datetime: range } - const searchBody = await es.constructSearchParams(queryParams, 1) + const searchBody = await db.constructSearchParams(queryParams, 1) // TODO: the ordering here is fragile. helper methods needed to ensure the queries are correct t.is( @@ -37,7 +37,7 @@ test('search datetime parameter intervals are correctly parsed', async (t) => { ] await datetimes.map(async ([datetime, start, end]) => { - const dtQuery = await es.buildDatetimeQuery({ datetime: datetime }) + const dtQuery = await db.buildDatetimeQuery({ datetime: datetime }) t.is(dtQuery.range['properties.datetime'].gte, start, 'datetime interval start') t.is(dtQuery.range['properties.datetime'].lte, end, 'datetime interval end') }) @@ -72,7 +72,7 @@ test('search datetime parameter instants are correctly parsed', async (t) => { ] await validDatetimes.map(async (datetime) => { - const dtQuery = await es.buildDatetimeQuery({ datetime: datetime }) + const dtQuery = await db.buildDatetimeQuery({ datetime: datetime }) t.is(dtQuery.term['properties.datetime'], datetime, 'datetime instant parses correctly') }) }) diff --git a/tests/unit/test-ingest-2.js b/tests/unit/test-ingest-2.js index 7b7118ef..c989e16a 100644 --- a/tests/unit/test-ingest-2.js +++ b/tests/unit/test-ingest-2.js @@ -3,7 +3,7 @@ const sinon = require('sinon') const MemoryStream = require('memorystream') const { ingestItems } = require('../../src/lib/ingest') const firstItem = require('../fixtures/stac/LC80100102015050LGN00.json') -const stream = require('../../src/lib/esStream') +const stream = require('../../src/lib/databaseStream') const setup = () => { const dupOptions = { @@ -16,22 +16,22 @@ const setup = () => { readable: false, objectMode: true } - // Catalog is filtered by real toEs transform stream but is left in here. - const toEs = new MemoryStream(null, dupOptions) - const esStream = new MemoryStream(null, writeOptions) + // Catalog is filtered by real toDB transform stream but is left in here. + const toDB = new MemoryStream(null, dupOptions) + const dbStream = new MemoryStream(null, writeOptions) const backend = { - stream: () => ({ toEs, esStream }), + stream: () => ({ toDB, dbStream }), prepare: sinon.stub().resolves(true) } return { - toEs, - esStream, + toDB, + dbStream, backend } } test.skip('ingestItem passes item through transform stream', async (t) => { - const { esStream } = setup() + const { dbStream } = setup() await ingestItems([firstItem], stream) - t.deepEqual(esStream.queue[0], firstItem) + t.deepEqual(dbStream.queue[0], firstItem) })