Merge pull request #409 from inaturalist/408-run-specs-with-docker

Run specs with docker compose - add tests related to user email/IP

Dockerfile

@@ -1,4 +1,6 @@
-FROM node:16
+# Platform should be forced to amd64
+# because node-mapnik is not available in arm64
+FROM --platform=linux/amd64 node:16 as base
 
 ENV NODE_ENV=development
 
@@ -10,6 +12,20 @@ COPY config_example.js config.js
 
 RUN npm install
 
+FROM base as test
+
+ENV NODE_ENV=test
+
+RUN apt-get update -qq && apt-get install -y postgresql-client-11
+
+COPY . .
+
+CMD [ "npm", "run", "coverage" ]
+
+FROM base as development
+
+ENV NODE_ENV=development
+
 COPY . .
 
 EXPOSE 4000

README.md

@@ -25,6 +25,29 @@ Filter by pattern: `NODE_ENV=test ./node_modules/mocha/bin/_mocha --recursive --
 
 You can also add `.only` to a `describe` or `it` call to only run that test when you run `npm test`, e.g. `it.only( "should only run this test" )`.
 
+# Running Tests with Docker
+
+You can run the tests with Docker Compose. 
+All required services will be started by the `docker-compose.test.yml` compose file:
+
+```
+docker compose -f docker-compose.test.yml up -d
+```
+
+You can follow the tests execution in the logs:
+
+```
+docker logs -f api-test
+```
+
+The first time you run the compose file, a local docker image for the API service will be automatically built, from you local GIT checkout. But if later you do some code changes, or update your GIT checkout, you need to re-build the docker image with:
+
+```
+docker compose -f docker-compose.test.yml build
+```
+
+This compose build is using the `test` target of the Dockerfile.
+
 # ESLint
 
 Please run ESLint to check for syntax formatting errors. To run ESLint, run: `npm run eslint`. Please address any syntax errors before submitting pull requests. ESLint will also run automatically via Github Actions on submitted pull requests along with tests.

config.js.ci

@@ -10,8 +10,7 @@ module.exports = {
     user: "postgres",
     host: "127.0.0.1",
     port: 5432,
-    geometry_field: "geom",
-    dbname: "inaturalist_test"
+    geometry_field: "geom"
   },
   websiteURL: "http://localhost:3000/",
   staticImagePrefix: "http://localhost:3000/attachments/",

docker-compose.test.yml

@@ -0,0 +1,70 @@
+version: "2"
+services:
+
+  redis:
+    container_name: redis
+    image: redis:6.0.3
+    ports:
+      - 6379:6379
+    volumes:
+      - redis_data_test:/data
+
+  memcached:
+    container_name: memcached
+    image: memcached:1.6.6
+    ports:
+      - 11211:11211
+
+  pg:
+    container_name: pg
+    image: postgis/postgis:12-3.0
+    environment:
+      POSTGRES_USER: 'inaturalist'
+      POSTGRES_PASSWORD: 'inaturalist'
+      POSTGRES_DB: inaturalist_test
+    ports:
+      - 5432:5432
+    volumes:
+      - pg_data_test:/var/lib/postgresql/data
+      - ./schema/database.sql:/docker-entrypoint-initdb.d/database.sql
+
+  es:
+    container_name: es
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.9.1
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    ports:
+      - 9200:9200
+    volumes:
+      - es_data_test:/usr/share/elasticsearch/data
+    command: >
+      /bin/sh -c "bin/elasticsearch-plugin list | grep -q analysis-kuromoji
+      || bin/elasticsearch-plugin install analysis-kuromoji;
+      /usr/local/bin/docker-entrypoint.sh"
+
+  api-test:
+    container_name: api-test
+    build: 
+      context: .
+      dockerfile: Dockerfile
+      target: test
+    environment:
+      NODE_ENV: test
+      INAT_DB_HOST: pg
+      INAT_DB_USER : 'inaturalist'
+      INAT_DB_PASS: 'inaturalist'
+      INAT_ES_HOST: es
+      INAT_REDIS_HOST: redis
+      INAT_WEB_HOST: host.docker.internal
+      INAT_DB_NAME: inaturalist_test
+      INAT_ES_INDEX_PREFIX: test
+    ports:
+      - 4000:4000
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  redis_data_test:
+  pg_data_test:
+  es_data_test:

lib/map_generator.js

@@ -30,6 +30,14 @@ const DEFAULT_POINTS_STYLE = `
 
 const MapGenerator = { merc: null };
 
+MapGenerator.buildDbName = ( ) => {
+  // Throw exception is NODE_ENV is not set
+  if ( _.isEmpty( process.env.NODE_ENV ) ) {
+    throw new Error( "env.NODE_ENV is not set" );
+  }
+  return process.env.NODE_ENV === "test" ? "inaturalist_test" : ( process.env.INAT_DB_NAME || `inaturalist_${process.env.NODE_ENV}` );
+};
+
 MapGenerator.blankImage = fs.readFileSync( path.join( __dirname, "assets/blank.png" ) );
 
 MapGenerator.createMercator = ( ) => {
@@ -63,6 +71,7 @@ MapGenerator.postgisDatasource = req => {
   const datasourceConfig = {
     ...config.database,
     ...( config.database.replica || { } ),
+    dbname: MapGenerator.buildDbName( ),
     type: "postgis",
     table: req.postgis.query,
     simplify_geometries: true,

lib/test_helper.js

@@ -1,5 +1,7 @@
+/* eslint-disable no-console */
 const fs = require( "fs" );
 const _ = require( "lodash" );
+const timersPromises = require( "timers/promises" );
 const Promise = require( "bluebird" );
 const { expect } = require( "chai" ); // eslint-disable-line import/no-extraneous-dependencies
 const sinon = require( "sinon" ); // eslint-disable-line import/no-extraneous-dependencies
@@ -8,6 +10,52 @@ const esClient = require( "./es_client" );
 
 const testHelper = { };
 
+testHelper.waitForPG = async numberOfAttempts => {
+  console.log( "Awaiting PG connection..." );
+  await timersPromises.setTimeout( 1000 );
+  try {
+    if ( await pgClient.connect( ) ) {
+      return true;
+    }
+  } catch ( e ) {
+    // Do nothing
+  }
+  numberOfAttempts -= 1;
+  if ( numberOfAttempts === 0 ) {
+    process.exit( );
+  }
+  return testHelper.waitForPG( numberOfAttempts );
+};
+
+testHelper.waitForES = async numberOfAttempts => {
+  console.log( "Awaiting ES connection..." );
+  await timersPromises.setTimeout( 1000 );
+  try {
+    await esClient.connect();
+    if ( esClient.connection ) {
+      if ( await esClient.connection.ping() ) {
+        return true;
+      }
+    }
+  } catch ( e ) {
+    // Do nothing
+  }
+  numberOfAttempts -= 1;
+  if ( numberOfAttempts === 0 ) {
+    process.exit( );
+  }
+  return testHelper.waitForES( numberOfAttempts );
+};
+
+testHelper.closePGConnection = async ( ) => {
+  await pgClient.connection.end( );
+  delete pgClient.connection;
+};
+
+testHelper.reconnectPGConnection = async ( ) => {
+  await pgClient.connect( );
+};
+
 testHelper.forEachIndex = async action => {
   if ( process.env.NODE_ENV !== "test" ) { return; }
   const settings = JSON.parse( fs.readFileSync( "schema/settings.js" ) );
@@ -108,6 +156,10 @@ testHelper.testInatJSNoPreload = async ( controller, endpoint, method, done ) =>
 };
 
 module.exports = {
+  waitForPG: testHelper.waitForPG,
+  waitForES: testHelper.waitForES,
+  closePGConnection: testHelper.closePGConnection,
+  reconnectPGConnection: testHelper.reconnectPGConnection,
   createIndices: testHelper.createIndices,
   deleteIndices: testHelper.deleteIndices,
   loadElasticsearchFixtures: testHelper.loadElasticsearchFixtures,