Split system tests into separate files

.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+  "recommendations": [
+    "davidanson.vscode-markdownlint",
+    "dbaeumer.vscode-eslint",
+    "timonwong.shellcheck"
+  ]
+}

.vscode/settings.json

@@ -1,5 +1,20 @@
 {
+  "editor.formatOnSave": true,
   "editor.rulers": [
     80
-  ]
+  ],
+  "eslint.format.enable": true,
+  "eslint.packageManager": "npm",
+  "files.trimTrailingWhitespace": true,
+  "[javascript]": {
+    "editor.defaultFormatter": "dbaeumer.vscode-eslint"
+  },
+  "[json]": {
+    "editor.defaultFormatter": "vscode.json-language-features"
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "dbaeumer.vscode-eslint"
+  },
+  "editor.tabSize": 2,
+  "files.insertFinalNewline": true
 }

README.md

@@ -1,4 +1,4 @@
-# stac-server 
+# stac-server
 
 ![](https://github.com/stac-utils/stac-server/workflows/Push%20Event/badge.svg)
 
@@ -140,9 +140,9 @@ npm run deploy -- --stage mystage --region eu-central-1
 Once deployed there is one final step - creating the indices and mappings in Elasticsearch. Invoke the `stac-server-<stage>-ingest` Lambda function with a payload of:
 
 ```json
-{ 
+{
   "create_indices": true
-} 
+}
 ```
 
 This can be done with the AWS CLI with (the final `-` parameter pipes the output to stdout):
@@ -192,7 +192,7 @@ Install [NVM](https://github.com/nvm-sh/nvm) to manage your Node.js environment.
 ```
 # uses version in .nvmrc
 nvm install
-nvm use 
+nvm use
 ```
 
 The package-lock.json was built with npm 8.5.0, so use at least this version.
@@ -258,7 +258,6 @@ When the system tests run, they:
 
 1. Wait for Elasticsearch to be available
 1. Delete all indices from Elasticsearch
-1. Add indices and test data to Elasticsearch
 1. Start an instance of the API. That API will be available at <http://localhost:3000/dev/>
 1. Wait for the API to be available
 1. Run the integration tests in `./tests/system/test_*.js`

bin/system-tests.sh

@@ -8,9 +8,6 @@ export AWS_SECRET_ACCESS_KEY='none'
 
 ./bin/wait-for-elasticsearch/run.sh
 
-echo "Setting up Elasticsearch"
-node ./tests/system/setup-es.js
-
 echo "Starting API"
 node ./src/lambdas/api/local.js >/dev/null 2>&2 &
 API_PID="$!"

package.json

@@ -46,6 +46,7 @@
     "lodash": "^4.17.21",
     "memorystream": "^0.3.1",
     "morgan": "^1.10.0",
+    "p-filter": "^2.0.0",
     "pump": "^3.0.0",
     "serverless-http": "^2.7.0",
     "through2": "^3.0.1"

tests/helpers/es.js

@@ -1,12 +1,33 @@
 // @ts-check
 
-const { connect } = require('../../src/lib/esClient')
+const { connect, createIndex } = require('../../src/lib/esClient')
 
+/**
+ * @returns {Promise<void>}
+ */
+const createCollectionsIndex = async () => {
+  await createIndex('collections')
+}
+
+/**
+ * @returns {Promise<void>}
+ */
 const refreshIndices = async () => {
   const esClient = await connect()
   await esClient.indices.refresh({ index: '_all' })
 }
 
+/**
+ * @returns {Promise<void>}
+ */
+const deleteAllIndices = async () => {
+  const es = await connect()
+  await es.indices.delete({ index: '_all' })
+  await refreshIndices()
+}
+
 module.exports = {
+  createCollectionsIndex,
+  deleteAllIndices,
   refreshIndices
 }

tests/helpers/ingest.js

@@ -0,0 +1,33 @@
+// @ts-check
+
+const awsClients = require('../../src/lib/aws-clients')
+const { handler } = require('../../src/lambdas/ingest')
+const { sqsTriggerLambda } = require('./sqs')
+const { nullLoggerContext } = require('./context')
+const { refreshIndices } = require('./es')
+
+/**
+ * @typedef {Object} IngestItemParams
+ * @property {string} ingestTopicArn
+ * @property {string} ingestQueueUrl
+ * @property {unknown} item
+ */
+
+/**
+ * @param {IngestItemParams} params
+ * @returns {Promise<void>}
+ */
+const ingestItem = async (params) => {
+  await awsClients.sns().publish({
+    TopicArn: params.ingestTopicArn,
+    Message: JSON.stringify(params.item)
+  }).promise()
+
+  await sqsTriggerLambda(params.ingestQueueUrl, handler, nullLoggerContext)
+
+  await refreshIndices()
+}
+
+module.exports = {
+  ingestItem
+}

tests/helpers/sns.js

@@ -0,0 +1,37 @@
+// @ts-check
+
+const awsClients = require('../../src/lib/aws-clients')
+const { randomId } = require('./utils')
+
+/**
+ * @returns {Promise<string>} topic ARN
+ */
+const createTopic = async () => {
+  const sns = awsClients.sns()
+
+  const { TopicArn } = await sns.createTopic({
+    Name: randomId('topic')
+  }).promise()
+
+  if (TopicArn) return TopicArn
+
+  throw new Error('Unable to create topic')
+}
+
+/**
+ * @param {string} topicArn
+ * @param {string} queueArn
+ * @returns {Promise<void>}
+ */
+const addSnsToSqsSubscription = async (topicArn, queueArn) => {
+  await awsClients.sns().subscribe({
+    TopicArn: topicArn,
+    Protocol: 'sqs',
+    Endpoint: queueArn
+  }).promise()
+}
+
+module.exports = {
+  addSnsToSqsSubscription,
+  createTopic
+}

tests/helpers/sqs.js

@@ -1,4 +1,6 @@
+const { isUndefined } = require('lodash')
 const awsClients = require('../../src/lib/aws-clients')
+const { randomId } = require('./utils')
 
 const sqsMessageToRecord = (message) => ({
   messageId: message.MessageId,
@@ -28,6 +30,52 @@ const sqsTriggerLambda = async (sqsUrl, handler, context = {}) => {
   return handler(event, context)
 }
 
+/**
+ * @param {string} url
+ * @returns {Promise<void>}
+ */
+const purgeQueue = async (url) => {
+  await awsClients.sqs().purgeQueue({ QueueUrl: url }).promise()
+}
+
+/**
+ * @returns {Promise<string>} the queue URL
+ */
+const createQueue = async () => {
+  const sqs = awsClients.sqs()
+
+  const { QueueUrl } = await sqs.createQueue({
+    QueueName: randomId('queue')
+  }).promise()
+
+  if (QueueUrl) return QueueUrl
+
+  throw new Error('Failed to create queue')
+}
+
+/**
+ * @param {string} queueUrl
+ * @returns {Promise<string>} queueArn
+ */
+const getQueueArn = async (queueUrl) => {
+  const sqs = awsClients.sqs()
+
+  const getQueueAttributesResult = await sqs.getQueueAttributes({
+    QueueUrl: queueUrl,
+    AttributeNames: ['QueueArn']
+  }).promise()
+
+  if (
+    isUndefined(getQueueAttributesResult.Attributes)
+    || isUndefined(getQueueAttributesResult.Attributes['QueueArn'])
+  ) throw new Error('Unable to get Queue ARN')
+
+  return getQueueAttributesResult.Attributes['QueueArn']
+}
+
 module.exports = {
+  createQueue,
+  getQueueArn,
+  purgeQueue,
   sqsTriggerLambda
 }

tests/helpers/system-tests.js

@@ -0,0 +1,47 @@
+// @ts-check
+
+const nock = require('nock')
+const { createCollectionsIndex, refreshIndices } = require('./es')
+const { createTopic, addSnsToSqsSubscription } = require('./sns')
+const { createQueue, getQueueArn } = require('./sqs')
+
+/**
+ * @typedef {Object} StandUpResult
+ * @property {string} ingestQueueUrl
+ * @property {string} ingestTopicArn
+ */
+
+/**
+ * @returns {Promise<StandUpResult>}
+ */
+const setup = async () => {
+  nock.disableNetConnect()
+  nock.enableNetConnect('localhost')
+
+  // Create Ingest SNS topic
+  const ingestTopicArn = await createTopic()
+
+  // Create SQS queue
+  const ingestQueueUrl = await createQueue()
+  const ingestQueueArn = await getQueueArn(ingestQueueUrl)
+
+  // Subscribe SQS queue to SNS topic
+  await addSnsToSqsSubscription(
+    ingestTopicArn,
+    ingestQueueArn
+  )
+
+  // Create ES collections index
+  await createCollectionsIndex()
+
+  await refreshIndices()
+
+  return {
+    ingestQueueUrl,
+    ingestTopicArn
+  }
+}
+
+module.exports = {
+  setup
+}

tests/helpers/types.d.ts

@@ -0,0 +1,4 @@
+export interface SystemTestContext {
+  ingestQueueUrl?: string
+  ingestTopicArn?: string
+}

tests/helpers/utils.js

@@ -3,7 +3,7 @@ const cryptoRandomString = require('crypto-random-string')
 const fs = require('fs')
 const path = require('path')
 
-const noop = () => {}
+const noop = () => { }
 
 const nullLogger = {
   debug: noop,