From ec5ccbbab58ba7d442992a3d8ffa8fb9a95379b8 Mon Sep 17 00:00:00 2001 From: sylvain-morin Date: Wed, 4 Oct 2023 16:21:59 +0200 Subject: [PATCH 1/4] Run specs with docker compose - add tests related to user email/IP --- config.js.ci | 3 +- docker-compose.test.yml | 70 +++++++++ lib/map_generator.js | 9 ++ lib/test_helper.js | 40 +++++ schema/fixtures.js | 196 ++++++++++++++++++++++++- schema/indices/users.js | 6 + test/_initialize.js | 58 -------- test/hooks.js | 65 +++++++- test/integration/v1/identifications.js | 88 +++++++++++ test/integration/v1/observations.js | 42 ++++++ test/integration/v1/projects.js | 101 +++++++++++++ test/integration/v1/search.js | 26 ++++ test/integration/v1/users.js | 47 ++++++ test/integration/v2/identifications.js | 13 ++ test/integration/v2/observations.js | 56 +++++++ test/integration/v2/projects.js | 40 +++++ test/integration/v2/search.js | 26 ++++ test/integration/v2/users.js | 63 ++++++++ 18 files changed, 887 insertions(+), 62 deletions(-) create mode 100644 docker-compose.test.yml delete mode 100644 test/_initialize.js diff --git a/config.js.ci b/config.js.ci index c0b9a7c2..7a4c237b 100644 --- a/config.js.ci +++ b/config.js.ci @@ -10,8 +10,7 @@ module.exports = { user: "postgres", host: "127.0.0.1", port: 5432, - geometry_field: "geom", - dbname: "inaturalist_test" + geometry_field: "geom" }, websiteURL: "http://localhost:3000/", staticImagePrefix: "http://localhost:3000/attachments/", diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..8879c6a1 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,70 @@ +version: "2" +services: + + redis: + container_name: redis + image: redis:6.0.3 + ports: + - 6379:6379 + volumes: + - redis_data_test:/data + + memcached: + container_name: memcached + image: memcached:1.6.6 + ports: + - 11211:11211 + + pg: + container_name: pg + image: postgis/postgis:12-3.0 + environment: + POSTGRES_USER: 'inaturalist' + POSTGRES_PASSWORD: 'inaturalist' + POSTGRES_DB: inaturalist_test + ports: + - 5432:5432 + volumes: + - pg_data_test:/var/lib/postgresql/data + - ./schema/database.sql:/docker-entrypoint-initdb.d/database.sql + + es: + container_name: es + image: docker.elastic.co/elasticsearch/elasticsearch:8.9.1 + environment: + - discovery.type=single-node + - xpack.security.enabled=false + ports: + - 9200:9200 + volumes: + - es_data_test:/usr/share/elasticsearch/data + command: > + /bin/sh -c "bin/elasticsearch-plugin list | grep -q analysis-kuromoji + || bin/elasticsearch-plugin install analysis-kuromoji; + /usr/local/bin/docker-entrypoint.sh" + + api-test: + container_name: api-test + build: . + environment: + NODE_ENV: test + INAT_DB_HOST: pg + INAT_DB_USER : 'inaturalist' + INAT_DB_PASS: 'inaturalist' + INAT_ES_HOST: es + INAT_REDIS_HOST: redis + INAT_WEB_HOST: host.docker.internal + INAT_DB_NAME: inaturalist_test + INAT_ES_INDEX_PREFIX: test + DB_ALREADY_INITIALIZED: true + ports: + - 4000:4000 + command: + /bin/sh -c "npm install ; npm run coverage" + extra_hosts: + - "host.docker.internal:host-gateway" + +volumes: + redis_data_test: + pg_data_test: + es_data_test: diff --git a/lib/map_generator.js b/lib/map_generator.js index 4e2ea164..ac3331cf 100644 --- a/lib/map_generator.js +++ b/lib/map_generator.js @@ -30,6 +30,14 @@ const DEFAULT_POINTS_STYLE = ` const MapGenerator = { merc: null }; +MapGenerator.buildDbName = ( ) => { + // Throw exception is NODE_ENV is not set + if ( _.isEmpty( process.env.NODE_ENV ) ) { + throw new Error( "env.NODE_ENV is not set" ); + } + return process.env.NODE_ENV === "test" ? "inaturalist_test" : ( process.env.INAT_DB_NAME || `inaturalist_${process.env.NODE_ENV}` ); +}; + MapGenerator.blankImage = fs.readFileSync( path.join( __dirname, "assets/blank.png" ) ); MapGenerator.createMercator = ( ) => { @@ -63,6 +71,7 @@ MapGenerator.postgisDatasource = req => { const datasourceConfig = { ...config.database, ...( config.database.replica || { } ), + dbname: MapGenerator.buildDbName( ), type: "postgis", table: req.postgis.query, simplify_geometries: true, diff --git a/lib/test_helper.js b/lib/test_helper.js index 4c647d32..6ef27453 100644 --- a/lib/test_helper.js +++ b/lib/test_helper.js @@ -1,5 +1,6 @@ const fs = require( "fs" ); const _ = require( "lodash" ); +const timersPromises = require( "timers/promises" ); const Promise = require( "bluebird" ); const { expect } = require( "chai" ); // eslint-disable-line import/no-extraneous-dependencies const sinon = require( "sinon" ); // eslint-disable-line import/no-extraneous-dependencies @@ -8,6 +9,43 @@ const esClient = require( "./es_client" ); const testHelper = { }; +testHelper.waitForPG = async numberOfAttempts => { + console.log( "Awaiting PG connection..." ); + await timersPromises.setTimeout( 1000 ); + try { + if ( await pgClient.connect( ) ) { + return true; + } + } catch ( e ) { + // Do nothing + } + numberOfAttempts -= 1; + if ( numberOfAttempts === 0 ) { + process.exit( ); + } + return testHelper.waitForPG( numberOfAttempts ); +}; + +testHelper.waitForES = async numberOfAttempts => { + console.log( "Awaiting ES connection..." ); + await timersPromises.setTimeout( 1000 ); + try { + await esClient.connect(); + if ( esClient.connection ) { + if ( await esClient.connection.ping() ) { + return true; + } + } + } catch ( e ) { + // Do nothing + } + numberOfAttempts -= 1; + if ( numberOfAttempts === 0 ) { + process.exit( ); + } + return testHelper.waitForES( numberOfAttempts ); +}; + testHelper.forEachIndex = async action => { if ( process.env.NODE_ENV !== "test" ) { return; } const settings = JSON.parse( fs.readFileSync( "schema/settings.js" ) ); @@ -108,6 +146,8 @@ testHelper.testInatJSNoPreload = async ( controller, endpoint, method, done ) => }; module.exports = { + waitForPG: testHelper.waitForPG, + waitForES: testHelper.waitForES, createIndices: testHelper.createIndices, deleteIndices: testHelper.deleteIndices, loadElasticsearchFixtures: testHelper.loadElasticsearchFixtures, diff --git a/schema/fixtures.js b/schema/fixtures.js index f2ef08ec..fb509a12 100644 --- a/schema/fixtures.js +++ b/schema/fixtures.js @@ -445,6 +445,45 @@ "id": 122, "login": "user122" } + }, + { + "id": 2023092501, + "uuid": "09130216-6201-11ee-8c99-0242ac120002", + "user": { + "id": 2023092501, + "login": "user2023092501", + "spam" : false, + "suspended" : false + }, + "body": "2023092501", + "category": "leading", + "current": true, + "current_taxon": true, + "taxon": { + "id": 5, + "uuid": "e6c0f90f-8527-4b56-a552-fe2273b61ec4", + "min_species_taxon_id": 5, + "is_active": true, + "iconic_taxon_id": 1, + "ancestor_ids": [1,2,3,4,5], + "min_species_ancestry": "1,2,3,4,5", + "rank_level": 10, + "min_species_ancestors": [ + { "id": 1 }, { "id": 2 }, { "id": 3 }, { "id": 4 }, { "id": 5 } + ] + }, + "observation": { + "id": 2023092501, + "user_id": 2023092501, + "taxon": { + "id": 5, + "iconic_taxon_id": 1, + "ancestor_ids": [1,2,3,4,5], + "min_species_taxon_id": 5, + "min_species_ancestry": "1,2,3,4,5", + "rank_level": 10 + } + } } ] }, @@ -1115,6 +1154,21 @@ "user": { "id": 2021121602 }, "created_at": "2021-12-08T01:00:00", "description": "Obs by a user who blocks user 2021121601" + }, + { + "id": 2023092501, + "uuid": "78e0b6e4-61fa-11ee-8c99-0242ac120002", + "user": { + "id": 2023092501, + "login": "user2023092501", + "spam" : false, + "suspended" : false + }, + "location": "50,50", + "private_location": "3,4", + "private_geojson": { "type": "Point", "coordinates": [ 4, 3 ] }, + "place_guess": "Tangerina", + "captive": true } ] }, @@ -1278,6 +1332,33 @@ "user_id": 2020100101, "user_ids": [2020100101, 2020100102], "prefers_user_trust": false + }, + { + "id": 2023092501, + "title": "project-2023092501", + "title_autocomplete": "project-2023092501", + "title_exact": "project-2023092501", + "slug": "project-2023092501", + "search_parameters": [ + { + "field": "place_id", + "value": 2 + } + ], + "search_parameter_fields": { + "place_id": 2 + }, + "user_id": 2023092501, + "user_ids": [2023092501, 2023092502, 2023092503], + "admins" : [ + { + "id" : 2023092501, + "user_id" : 2023092501, + "project_id" : 2023092501, + "role" : "manager" + } + ], + "prefers_user_trust": false } ] }, @@ -1810,6 +1891,27 @@ "id": 2021121602, "login": "user2021121602", "name": "User that blocks user2021121601" + }, + { + "id": 2023092501, + "login": "user2023092501", + "name": "User2023092501 with email and IP", + "email": "user2023092501@gmail.com", + "last_ip": "192.168.0.1" + }, + { + "id": 2023092502, + "login": "user2023092502", + "name": "User2023092502 with email and IP", + "email": "user2023092502@gmail.com", + "last_ip": "192.168.0.2" + }, + { + "id": 2023092503, + "login": "user2023092503", + "name": "User2023092503 with email and IP", + "email": "user2023092503@gmail.com", + "last_ip": "192.168.0.3" } ] }, @@ -2074,6 +2176,14 @@ "category": "leading", "user_id": 122, "observation_id": 13 + }, + { + "id": 2023092501, + "current": true, + "created_at": "2023-09-25 00:00:00", + "category": "leading", + "user_id": 2023092501, + "observation_id": 2023092501 } ], "lists": [ @@ -2338,6 +2448,10 @@ { "id": 2021121601, "uuid": "b568967d-4f0e-430c-9cb9-e28db5004c37" + }, + { + "id": 2023092501, + "uuid": "78e0b6e4-61fa-11ee-8c99-0242ac120002" } ], "observation_photos": [ @@ -2657,6 +2771,23 @@ "id": 2020100102, "project_id": 2020100101, "user_id": 2020100102 + }, + { + "id": 2023092501, + "project_id": 2023092501, + "user_id": 2023092501, + "role": "manager" + }, + { + "id": 2023092502, + "project_id": 2023092501, + "user_id": 2023092502, + "role": "curator" + }, + { + "id": 2023092503, + "project_id": 2023092501, + "user_id": 2023092503 } ], "projects": [ @@ -2692,6 +2823,12 @@ "project_type": "collection", "title": "Redundant Observations in Massachusetts With Disabled Trust", "slug": "redundant-observations-in-massachusetts-with-disabled-trust" + }, + { + "id": 2023092501, + "slug": "project-2023092501", + "title": "project-2023092501", + "user_id": 2023092501 } ], "provider_authorizations": [ @@ -2723,6 +2860,33 @@ "title": "post 2 title", "body": "post 2 body2", "user_id": 1 + }, + { + "id": 2023092501, + "parent_id": 2023092501, + "parent_type": "Project", + "published_at": "2023-09-25 02:00:00", + "title": "2023092501 title", + "body": "2023092501 body", + "user_id": 2023092501 + }, + { + "id": 2023092502, + "parent_id": 2023092501, + "parent_type": "Project", + "published_at": "2023-09-25 02:00:00", + "title": "2023092502 title", + "body": "2023092502 body", + "user_id": 2023092502 + }, + { + "id": 2023092503, + "parent_id": 2023092501, + "parent_type": "Project", + "published_at": "2023-09-25 02:00:00", + "title": "2023092503 title", + "body": "2023092503 body", + "user_id": 2023092503 } ], "roles": [ @@ -2963,7 +3127,7 @@ "name": "username", "site_id": 1, "description": "a very original user", - "last_active": "2022-03-01" + "last_active": "2022-03-01" }, { "id": 5, @@ -3072,6 +3236,36 @@ "login": "user2021121602", "name": "User that blocks user2021121601", "description": "User that blocks user2021121601" + }, + { + "id": 2023092501, + "login": "user2023092501", + "name": "User2023092501 with email and IP", + "email": "user2023092501@gmail.com", + "last_ip": "192.168.0.1", + "created_at": "2020-01-01", + "updated_at": "2020-01-01", + "last_active": "2023-01-01" + }, + { + "id": 2023092502, + "login": "user2023092502", + "name": "User2023092502 with email and IP", + "email": "user2023092502@gmail.com", + "last_ip": "192.168.0.2", + "created_at": "2020-01-01", + "updated_at": "2020-01-01", + "last_active": "2023-01-01" + }, + { + "id": 2023092503, + "login": "user2023092503", + "name": "User2023092503 with email and IP", + "email": "user2023092503@gmail.com", + "last_ip": "192.168.0.3", + "created_at": "2020-01-01", + "updated_at": "2020-01-01", + "last_active": "2023-01-01" } ], "user_blocks": [ diff --git a/schema/indices/users.js b/schema/indices/users.js index 54c61204..cf43ef54 100644 --- a/schema/indices/users.js +++ b/schema/indices/users.js @@ -77,6 +77,12 @@ }, "uuid": { "type": "keyword" + }, + "email": { + "type": "keyword" + }, + "last_ip": { + "type": "keyword" } } } \ No newline at end of file diff --git a/test/_initialize.js b/test/_initialize.js deleted file mode 100644 index bd2ddebe..00000000 --- a/test/_initialize.js +++ /dev/null @@ -1,58 +0,0 @@ -/* eslint-disable no-console */ - -const { execSync } = require( "child_process" ); -const config = require( "../config" ); - -console.log( "INITIALIZING TEST ENVIRONMENT\n" ); - -// For tests, we want to make absolutely sure the test database is clean and -// new, and we need to make sure that happens before we try to connect to it, -// which happens every time we require pg_client, which happens inside of -// test_helper, so here we're setting up the database *before* we require -// anything that would require those modules. Naming this file _initialize.js -// ensures mocha runs this file first when running tests. -const dbname = "inaturalist_test"; -const testDbConnectionVar = `${config.database.host ? `PGHOST=${config.database.host}` : ""} \ - ${config.database.user ? `PGUSER=${config.database.user}` : ""} \ - ${config.database.password ? `PGPASSWORD=${config.database.password}` : ""}`; - -console.log( "Dropping existing test database" ); -execSync( `${testDbConnectionVar} dropdb --if-exists ${dbname}`, { stdio: [0, 1, 2] } ); -console.log( "Creating test database" ); -execSync( `${testDbConnectionVar} createdb -O ${config.database.user} ${dbname}`, { stdio: [0, 1, 2] } ); -console.log( "Loading test database schema" ); -execSync( `${testDbConnectionVar} psql -q -f schema/database.sql -d ${dbname}`, { stdio: [0, 1, 2] } ); - -/* eslint import/order: 0 */ -const testHelper = require( "../lib/test_helper" ); -const inaturalistjs = require( "inaturalistjs" ); -const Taxon = require( "../lib/models/taxon" ); - -// Note, these are mocha callbacks called at a global scope, so these run before -// or after *all* tests. Nothing special about initialize.js, it's just another -// test that specifies these before and after callbacks for all tests. - -before( async function ( ) { - this.timeout( 20000 ); - console.log( "Creating ES indices" ); - await testHelper.createIndices( ); - console.log( "Loading ES fixtures" ); - await testHelper.loadElasticsearchFixtures( ); - console.log( "Loading Postgres fixtures" ); - await testHelper.loadPostgresqlFixtures( ); - console.log( "Loading iconic taxa" ); - await Taxon.loadIconicTaxa( ); - console.log( "\nDONE INITIALIZING TEST ENVIRONMENT\n\n" ); -} ); - -after( async function ( ) { - this.timeout( 10000 ); - await testHelper.deleteIndices( ); -} ); - -beforeEach( ( ) => { - inaturalistjs.setConfig( { - apiURL: "http://localhost:3000", - writeApiURL: "http://localhost:3000" - } ); -} ); diff --git a/test/hooks.js b/test/hooks.js index 532e5369..46ff92b9 100644 --- a/test/hooks.js +++ b/test/hooks.js @@ -1,10 +1,73 @@ const { expect } = require( "chai" ); +const { execSync } = require( "child_process" ); +const inaturalistjs = require( "inaturalistjs" ); const app = require( "../app" ); +const testHelper = require( "../lib/test_helper" ); +const Taxon = require( "../lib/models/taxon" ); +const config = require( "../config" ); + +function initializeDb() { + // For tests, we want to make absolutely sure the test database is clean and + // new, and we need to make sure that happens before we try to connect to it, + // which happens every time we require pg_client, which happens inside of + // test_helper, so here we're setting up the database *before* we require + // anything that would require those modules. Naming this file _initialize.js + // ensures mocha runs this file first when running tests. + const dbname = "inaturalist_test"; + const testDbConnectionVar = `${config.database.host ? `PGHOST=${config.database.host}` : ""} \ + ${config.database.user ? `PGUSER=${config.database.user}` : ""} \ + ${config.database.password ? `PGPASSWORD=${config.database.password}` : ""}`; + + console.log( "Dropping existing test database" ); + execSync( `${testDbConnectionVar} dropdb --if-exists ${dbname}`, { stdio: [0, 1, 2] } ); + console.log( "Creating test database" ); + execSync( `${testDbConnectionVar} createdb -O ${config.database.user} ${dbname}`, { stdio: [0, 1, 2] } ); + console.log( "Loading test database schema" ); + execSync( `${testDbConnectionVar} psql -q -f schema/database.sql -d ${dbname}`, { stdio: [0, 1, 2] } ); +} + +exports.mochaGlobalSetup = async function () { + expect( process.env.NODE_ENV ).to.eq( "test" ); + + // Wait for Postgres + console.log( "Waiting for Postgres..." ); + await testHelper.waitForPG( 100 ); + + // Wait for ES + console.log( "Waiting for ElasticSearch..." ); + await testHelper.waitForES( 100 ); + + console.log( "\n\nINITIALIZING TEST ENVIRONMENT\n\n" ); + + if ( !process.env.DB_ALREADY_INITIALIZED ) { + initializeDb( ); + } + + console.log( "Creating ES indices" ); + await testHelper.createIndices( ); + console.log( "Loading ES fixtures" ); + await testHelper.loadElasticsearchFixtures( ); + console.log( "Loading Postgres fixtures" ); + await testHelper.loadPostgresqlFixtures( ); + console.log( "Loading iconic taxa" ); + await Taxon.loadIconicTaxa( ); + + console.log( "\n\nDONE INITIALIZING TEST ENVIRONMENT\n\n" ); +}; exports.mochaHooks = { async beforeAll( ) { - expect( process.env.NODE_ENV ).to.eq( "test" ); this.timeout( 20000 ); this.app = await app( ); + }, + async afterAll( ) { + this.timeout( 10000 ); + await testHelper.deleteIndices( ); + }, + async beforeEach( ) { + inaturalistjs.setConfig( { + apiURL: "http://localhost:3000", + writeApiURL: "http://localhost:3000" + } ); } }; diff --git a/test/integration/v1/identifications.js b/test/integration/v1/identifications.js index 5788b785..ccac585f 100644 --- a/test/integration/v1/identifications.js +++ b/test/integration/v1/identifications.js @@ -36,6 +36,94 @@ describe( "Identifications", ( ) => { expect( result.id ).to.eq( 102 ); } ).expect( 200, done ); } ); + + it( "never returns email or IP for user in identification", function ( done ) { + request( this.app ).get( "/v1/identifications?id=2023092501" ) + .expect( res => { + const identification = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( identification.id ).to.eq( 2023092501 ); + expect( identification.user ).not.to.be.undefined; + expect( identification.user.id ).to.eq( 2023092501 ); + expect( identification.user.email ).to.be.undefined; + expect( identification.user.last_ip ).to.be.undefined; + expect( identification.observation ).not.to.be.undefined; + expect( identification.observation.user ).not.to.be.undefined; + expect( identification.observation.user.id ).to.eq( 2023092501 ); + expect( identification.observation.user.email ).to.be.undefined; + expect( identification.observation.user.last_ip ).to.be.undefined; + expect( identification.observation.identifications ).not.to.be.undefined; + expect( identification.observation.identifications.length ).to.eq( 1 ); + expect( identification.observation.identifications[0].user ).not.to.be.undefined; + expect( identification.observation.identifications[0].user.id ).to.eq( 2023092501 ); + expect( identification.observation.identifications[0].user.email ).to.be.undefined; + expect( identification.observation.identifications[0].user.last_ip ).to.be.undefined; + } ).expect( 200, done ); + } ); + } ); + + describe( "details", ( ) => { + it( "never returns email or IP for user in identification", function ( done ) { + request( this.app ).get( "/v1/identifications/2023092501" ) + .expect( res => { + const identification = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( identification.id ).to.eq( 2023092501 ); + expect( identification.user ).not.to.be.undefined; + expect( identification.user.id ).to.eq( 2023092501 ); + expect( identification.user.email ).to.be.undefined; + expect( identification.user.last_ip ).to.be.undefined; + expect( identification.observation ).not.to.be.undefined; + expect( identification.observation.user ).not.to.be.undefined; + expect( identification.observation.user.id ).to.eq( 2023092501 ); + expect( identification.observation.user.email ).to.be.undefined; + expect( identification.observation.user.last_ip ).to.be.undefined; + expect( identification.observation.identifications ).not.to.be.undefined; + expect( identification.observation.identifications.length ).to.eq( 1 ); + expect( identification.observation.identifications[0].user ).not.to.be.undefined; + expect( identification.observation.identifications[0].user.id ).to.eq( 2023092501 ); + expect( identification.observation.identifications[0].user.email ).to.be.undefined; + expect( identification.observation.identifications[0].user.last_ip ).to.be.undefined; + } ).expect( 200, done ); + } ); + } ); + + describe( "identifiers", ( ) => { + it( "never returns email or IP for user in identification", function ( done ) { + request( this.app ).get( "/v1/identifications/identifiers?id=2023092501" ) + .expect( res => { + const identification = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( identification.user_id ).to.eq( 2023092501 ); + expect( identification.user ).not.to.be.undefined; + expect( identification.user.id ).to.eq( 2023092501 ); + expect( identification.user.email ).to.be.undefined; + expect( identification.user.last_ip ).to.be.undefined; + } ).expect( 200, done ); + } ); + } ); + + describe( "observers", ( ) => { + it( "never returns email or IP for user in identification", function ( done ) { + request( this.app ).get( "/v1/identifications/observers?id=2023092501" ) + .expect( res => { + const identification = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( identification.user_id ).to.eq( 2023092501 ); + expect( identification.user ).not.to.be.undefined; + expect( identification.user.id ).to.eq( 2023092501 ); + expect( identification.user.email ).to.be.undefined; + expect( identification.user.last_ip ).to.be.undefined; + } ).expect( 200, done ); + } ); } ); describe( "species_counts", ( ) => { diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 52bf2967..bff8fd7c 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -210,6 +210,27 @@ describe( "Observations", ( ) => { .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in observation", function ( done ) { + request( this.app ).get( "/v1/observations/2023092501" ).expect( res => { + const observation = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( observation.id ).to.eq( 2023092501 ); + expect( observation.user ).not.to.be.undefined; + expect( observation.user.id ).to.eq( 2023092501 ); + expect( observation.user.email ).to.be.undefined; + expect( observation.user.last_ip ).to.be.undefined; + expect( observation.identifications ).not.to.be.undefined; + expect( observation.identifications.length ).to.eq( 1 ); + expect( observation.identifications[0].user ).not.to.be.undefined; + expect( observation.identifications[0].user.id ).to.eq( 2023092501 ); + expect( observation.identifications[0].user.email ).to.be.undefined; + expect( observation.identifications[0].user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "create", ( ) => { @@ -1223,6 +1244,27 @@ describe( "Observations", ( ) => { .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in observation", function ( done ) { + request( this.app ).get( "/v1/observations?id=2023092501" ).expect( res => { + const observation = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( observation.id ).to.eq( 2023092501 ); + expect( observation.user ).not.to.be.undefined; + expect( observation.user.id ).to.eq( 2023092501 ); + expect( observation.user.email ).to.be.undefined; + expect( observation.user.last_ip ).to.be.undefined; + expect( observation.identifications ).not.to.be.undefined; + expect( observation.identifications.length ).to.eq( 1 ); + expect( observation.identifications[0].user ).not.to.be.undefined; + expect( observation.identifications[0].user.id ).to.eq( 2023092501 ); + expect( observation.identifications[0].user.email ).to.be.undefined; + expect( observation.identifications[0].user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "histogram", ( ) => { diff --git a/test/integration/v1/projects.js b/test/integration/v1/projects.js index a9611c5d..03d10e3c 100644 --- a/test/integration/v1/projects.js +++ b/test/integration/v1/projects.js @@ -70,6 +70,25 @@ describe( "Projects", ( ) => { } ) .expect( 200, done ); } ); + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v1/projects?q=project-2023092501" ) + .expect( res => { + const project = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( project.id ).to.eq( 2023092501 ); + expect( project.admins ).not.to.be.undefined; + expect( project.admins[0] ).not.to.be.undefined; + expect( project.admins[0].user ).not.to.be.undefined; + expect( project.admins[0].user.email ).to.be.undefined; + expect( project.admins[0].user.last_ip ).to.be.undefined; + expect( project.user ).not.to.be.undefined; + expect( project.user.email ).to.be.undefined; + expect( project.user.last_ip ).to.be.undefined; + } ) + .expect( 200, done ); + } ); } ); describe( "show", ( ) => { @@ -110,6 +129,27 @@ describe( "Projects", ( ) => { } ).expect( "Content-Type", /json/ ) .expect( 422, done ); } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v1/projects/2023092501" ) + .expect( res => { + const project = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( project.id ).to.eq( 2023092501 ); + expect( project.admins ).not.to.be.undefined; + expect( project.admins[0] ).not.to.be.undefined; + expect( project.admins[0].user ).not.to.be.undefined; + expect( project.admins[0].user.email ).to.be.undefined; + expect( project.admins[0].user.last_ip ).to.be.undefined; + expect( project.user ).not.to.be.undefined; + expect( project.user.email ).to.be.undefined; + expect( project.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "autocomplete", ( ) => { @@ -182,6 +222,27 @@ describe( "Projects", ( ) => { } ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v1/projects/autocomplete?q=2023092501" ) + .expect( res => { + const project = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( project.id ).to.eq( 2023092501 ); + expect( project.admins ).not.to.be.undefined; + expect( project.admins[0] ).not.to.be.undefined; + expect( project.admins[0].user ).not.to.be.undefined; + expect( project.admins[0].user.email ).to.be.undefined; + expect( project.admins[0].user.last_ip ).to.be.undefined; + expect( project.user ).not.to.be.undefined; + expect( project.user.email ).to.be.undefined; + expect( project.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "members", ( ) => { @@ -233,6 +294,29 @@ describe( "Projects", ( ) => { } ).expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v1/projects/2023092501/members" ) + .expect( res => { + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 3 ); + expect( res.body.total_results ).to.eq( 3 ); + expect( res.body.results.length ).to.eq( 3 ); + expect( res.body.results[0].user.id ).to.eq( 2023092501 ); + expect( res.body.results[0].user ).not.to.be.undefined; + expect( res.body.results[0].user.email ).to.be.undefined; + expect( res.body.results[0].user.last_ip ).to.be.undefined; + expect( res.body.results[1].user.id ).to.eq( 2023092502 ); + expect( res.body.results[1].user ).not.to.be.undefined; + expect( res.body.results[1].user.email ).to.be.undefined; + expect( res.body.results[1].user.last_ip ).to.be.undefined; + expect( res.body.results[2].user.id ).to.eq( 2023092503 ); + expect( res.body.results[2].user ).not.to.be.undefined; + expect( res.body.results[2].user.email ).to.be.undefined; + expect( res.body.results[2].user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "posts", ( ) => { @@ -268,6 +352,23 @@ describe( "Projects", ( ) => { .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v1/projects/2023092501/posts" ) + .expect( res => { + expect( res.body.total_results ).to.eq( 3 ); + expect( res.body.results[0].user ).not.to.be.undefined; + expect( res.body.results[0].user.email ).to.be.undefined; + expect( res.body.results[0].user.last_ip ).to.be.undefined; + expect( res.body.results[1].user ).not.to.be.undefined; + expect( res.body.results[1].user.email ).to.be.undefined; + expect( res.body.results[1].user.last_ip ).to.be.undefined; + expect( res.body.results[2].user ).not.to.be.undefined; + expect( res.body.results[2].user.email ).to.be.undefined; + expect( res.body.results[2].user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "subscriptions", ( ) => { diff --git a/test/integration/v1/search.js b/test/integration/v1/search.js index 5b9596ea..93c25c61 100644 --- a/test/integration/v1/search.js +++ b/test/integration/v1/search.js @@ -58,5 +58,31 @@ describe( "Search", ( ) => { } ).expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user", function ( done ) { + request( this.app ).get( "/v1/search?q=User2023092501" ).expect( res => { + const user = res.body.results[0]; + expect( user.type ).to.eq( "User" ); + expect( user.record ).not.to.be.undefined; + expect( user.record.id ).to.eq( 2023092501 ); + expect( user.record.email ).to.be.undefined; + expect( user.record.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v1/search?q=project-2023092501" ).expect( res => { + const project = res.body.results[0]; + expect( project.type ).to.eq( "Project" ); + expect( project.record ).not.to.be.undefined; + expect( project.record.id ).to.eq( 2023092501 ); + expect( project.record.admins ).not.to.be.undefined; + expect( project.record.admins[0] ).not.to.be.undefined; + expect( project.record.admins[0].email ).to.be.undefined; + expect( project.record.admins[0].last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); } ); diff --git a/test/integration/v1/users.js b/test/integration/v1/users.js index 71369cb2..78e145c3 100644 --- a/test/integration/v1/users.js +++ b/test/integration/v1/users.js @@ -39,6 +39,21 @@ describe( "Users", ( ) => { request( this.app ).get( "/v1/users/123123" ) .expect( "Content-Type", /json/ ).expect( 404, done ); } ); + + it( "never returns email or IP for user", function ( done ) { + request( this.app ).get( "/v1/users/2023092501" ) + .expect( res => { + const user = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( user.id ).to.eq( 2023092501 ); + expect( user ).to.not.have.property( "email" ); + expect( user ).to.not.have.property( "last_ip" ); + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "update", ( ) => { @@ -96,6 +111,20 @@ describe( "Users", ( ) => { } ).expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user", function ( done ) { + request( this.app ).get( "/v1/users/autocomplete?q=user2023092501" ) + .expect( res => { + const user = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( user.id ).to.eq( 2023092501 ); + expect( user ).to.not.have.property( "email" ); + expect( user ).to.not.have.property( "last_ip" ); + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "projects", ( ) => { @@ -121,6 +150,24 @@ describe( "Users", ( ) => { } ).expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v1/users/2023092501/projects" ) + .expect( res => { + expect( res.body.page ).to.eq( 1 ); + const project = _.find( res.body.results, p => p.slug === "project-2023092501" ); + expect( project ).not.to.be.undefined; + expect( project.admins ).not.to.be.undefined; + expect( project.admins[0] ).not.to.be.undefined; + expect( project.admins[0].user ).not.to.be.undefined; + expect( project.admins[0].user.email ).to.be.undefined; + expect( project.admins[0].user.last_ip ).to.be.undefined; + expect( project.user ).not.to.be.undefined; + expect( project.user.email ).to.be.undefined; + expect( project.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "me", ( ) => { diff --git a/test/integration/v2/identifications.js b/test/integration/v2/identifications.js index 0f343be6..350fd459 100644 --- a/test/integration/v2/identifications.js +++ b/test/integration/v2/identifications.js @@ -52,6 +52,19 @@ describe( "Identifications", ( ) => { .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + it( "never returns email or IP for user in identification", function ( done ) { + request( this.app ).get( "/v2/identifications/identifiers?per_page=100&fields=all" ) + .expect( res => { + expect( res.body.page ).to.eq( 1 ); + const record = _.find( res.body.results, u => u.user_id === 2023092501 ); + expect( record ).not.to.be.undefined; + expect( record.user ).not.to.be.undefined; + expect( record.user.id ).eq( 2023092501 ); + expect( record.user.email ).to.be.undefined; + expect( record.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "update", ( ) => { diff --git a/test/integration/v2/observations.js b/test/integration/v2/observations.js index cc0cdf72..0dc036e4 100644 --- a/test/integration/v2/observations.js +++ b/test/integration/v2/observations.js @@ -144,6 +144,27 @@ describe( "Observations", ( ) => { .expect( 200, done ); } ); + it( "never returns email or IP for user in observation", function ( done ) { + request( this.app ).get( "/v2/observations/78e0b6e4-61fa-11ee-8c99-0242ac120002?fields=all" ).expect( res => { + const observation = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( observation.id ).to.eq( 2023092501 ); + expect( observation.user ).not.to.be.undefined; + expect( observation.user.id ).to.eq( 2023092501 ); + expect( observation.user.email ).to.be.undefined; + expect( observation.user.last_ip ).to.be.undefined; + expect( observation.identifications ).not.to.be.undefined; + expect( observation.identifications.length ).to.eq( 1 ); + expect( observation.identifications[0].user ).not.to.be.undefined; + expect( observation.identifications[0].user.id ).to.eq( 2023092501 ); + expect( observation.identifications[0].user.email ).to.be.undefined; + expect( observation.identifications[0].user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + // the observations.show endpoint should use the ES mget method to fetch observations for the // show endpoint, unless there are additional parameters that will filter the observations // returned. This is because the mget method is not affected by the normal ES refresh cycle, @@ -371,6 +392,27 @@ describe( "Observations", ( ) => { } ) .expect( 500, done ); } ); + + it( "never returns email or IP for user in observation", function ( done ) { + request( this.app ).get( "/v2/observations?id=2023092501&fields=all" ).expect( res => { + const observation = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( observation.id ).to.eq( 2023092501 ); + expect( observation.user ).not.to.be.undefined; + expect( observation.user.id ).to.eq( 2023092501 ); + expect( observation.user.email ).to.be.undefined; + expect( observation.user.last_ip ).to.be.undefined; + expect( observation.identifications ).not.to.be.undefined; + expect( observation.identifications.length ).to.eq( 1 ); + expect( observation.identifications[0].user ).not.to.be.undefined; + expect( observation.identifications[0].user.id ).to.eq( 2023092501 ); + expect( observation.identifications[0].user.email ).to.be.undefined; + expect( observation.identifications[0].user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "create", ( ) => { @@ -585,6 +627,20 @@ describe( "Observations", ( ) => { .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in observation", function ( done ) { + request( this.app ).get( "/v2/observations/observers?user_id=2023092501&fields=all" ).expect( res => { + const observation = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( observation.user ).not.to.be.undefined; + expect( observation.user.id ).to.eq( 2023092501 ); + expect( observation.user.email ).to.be.undefined; + expect( observation.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "popularFieldValues", ( ) => { diff --git a/test/integration/v2/projects.js b/test/integration/v2/projects.js index de570897..75e1ce24 100644 --- a/test/integration/v2/projects.js +++ b/test/integration/v2/projects.js @@ -46,6 +46,26 @@ describe( "Projects", ( ) => { } ) .expect( 200, done ); } ); + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v2/projects/2023092501?fields=all" ) + .expect( res => { + const p = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( p.id ).to.eq( 2023092501 ); + expect( p.admins ).not.to.be.undefined; + expect( p.admins[0] ).not.to.be.undefined; + expect( p.admins[0].user ).not.to.be.undefined; + expect( p.admins[0].user.email ).to.be.undefined; + expect( p.admins[0].user.last_ip ).to.be.undefined; + expect( p.user ).not.to.be.undefined; + expect( p.user.email ).to.be.undefined; + expect( p.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "search", ( ) => { @@ -79,5 +99,25 @@ describe( "Projects", ( ) => { } ) .expect( 200, done ); } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v2/projects?q=project-2023092501&fields=all" ) + .expect( res => { + const project = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( project.id ).to.eq( 2023092501 ); + expect( project.admins ).not.to.be.undefined; + expect( project.admins[0] ).not.to.be.undefined; + expect( project.admins[0].user ).not.to.be.undefined; + expect( project.admins[0].user.email ).to.be.undefined; + expect( project.admins[0].user.last_ip ).to.be.undefined; + expect( project.user ).not.to.be.undefined; + expect( project.user.email ).to.be.undefined; + expect( project.user.last_ip ).to.be.undefined; + } ) + .expect( 200, done ); + } ); } ); } ); diff --git a/test/integration/v2/search.js b/test/integration/v2/search.js index e651fa6b..4ef5fc91 100644 --- a/test/integration/v2/search.js +++ b/test/integration/v2/search.js @@ -58,5 +58,31 @@ describe( "Search", ( ) => { } ).expect( "Content-Type", /json/ ) .expect( 200, done ); } ); + + it( "never returns email or IP for user", function ( done ) { + request( this.app ).get( "/v2/search?q=User2023092501&fields=all" ).expect( res => { + const record = res.body.results[0]; + expect( record.type ).to.eq( "user" ); + expect( record.user ).not.to.be.undefined; + expect( record.user.id ).to.eq( 2023092501 ); + expect( record.user.email ).to.be.undefined; + expect( record.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v2/search?q=project-2023092501&fields=all" ).expect( res => { + const record = res.body.results[0]; + expect( record.type ).to.eq( "project" ); + expect( record.project ).not.to.be.undefined; + expect( record.project.id ).to.eq( 2023092501 ); + expect( record.project.admins ).not.to.be.undefined; + expect( record.project.admins[0] ).not.to.be.undefined; + expect( record.project.admins[0].email ).to.be.undefined; + expect( record.project.admins[0].last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); } ); diff --git a/test/integration/v2/users.js b/test/integration/v2/users.js index 67aadf79..7189081c 100644 --- a/test/integration/v2/users.js +++ b/test/integration/v2/users.js @@ -40,7 +40,38 @@ describe( "Users", ( ) => { } ) .expect( 200, done ); } ); + it( "never returns email or IP for user", function ( done ) { + request( this.app ).get( "/v2/users/2023092501?fields=all" ) + .expect( res => { + const user = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( res.body.results.length ).to.eq( 1 ); + expect( user.id ).to.eq( 2023092501 ); + expect( user ).to.not.have.property( "email" ); + expect( user ).to.not.have.property( "last_ip" ); + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); + + describe( "autocomplete", ( ) => { + it( "never returns email or IP for user", function ( done ) { + request( this.app ).get( "/v2/users/autocomplete?q=user2023092501&fields=all" ) + .expect( res => { + const user = res.body.results[0]; + expect( res.body.page ).to.eq( 1 ); + expect( res.body.per_page ).to.eq( 1 ); + expect( res.body.total_results ).to.eq( 1 ); + expect( user.id ).to.eq( 2023092501 ); + expect( user ).to.not.have.property( "email" ); + expect( user ).to.not.have.property( "last_ip" ); + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + } ); + describe( "update_session", ( ) => { it( "should fail without auth", function ( done ) { request( this.app ).put( "/v2/users/update_session" ) @@ -70,6 +101,27 @@ describe( "Users", ( ) => { .expect( 204, done ); } ); } ); + + describe( "projects", ( ) => { + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v2/users/2023092501/projects?fields=all" ) + .expect( res => { + expect( res.body.page ).to.eq( 1 ); + const project = _.find( res.body.results, p => p.slug === "project-2023092501" ); + expect( project ).not.to.be.undefined; + expect( project.admins ).not.to.be.undefined; + expect( project.admins[0] ).not.to.be.undefined; + expect( project.admins[0].user ).not.to.be.undefined; + expect( project.admins[0].user.email ).to.be.undefined; + expect( project.admins[0].user.last_ip ).to.be.undefined; + expect( project.user ).not.to.be.undefined; + expect( project.user.email ).to.be.undefined; + expect( project.user.last_ip ).to.be.undefined; + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + } ); + describe( "index", ( ) => { it( "should return JSON", function ( done ) { request( this.app ).get( "/v2/users?fields=login" ) @@ -162,6 +214,17 @@ describe( "Users", ( ) => { } ) .expect( 200, done ); } ); + it( "never returns email or IP for user in project", function ( done ) { + request( this.app ).get( "/v2/users?fields=all&per_page=100" ) + .expect( res => { + const user = _.find( res.body.results, u => u.id === 2023092501 ); + expect( user ).not.to.be.undefined; + expect( user.id ).to.eq( 2023092501 ); + expect( user ).to.not.have.property( "email" ); + expect( user ).to.not.have.property( "last_ip" ); + } ).expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); } ); describe( "update", ( ) => { From 88304d45db42db238ec442c0455645427bc8ac7f Mon Sep 17 00:00:00 2001 From: sylvain-morin Date: Wed, 4 Oct 2023 17:09:03 +0200 Subject: [PATCH 2/4] Run specs with docker compose - add tests related to user email/IP --- test/hooks.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/hooks.js b/test/hooks.js index 46ff92b9..039c6392 100644 --- a/test/hooks.js +++ b/test/hooks.js @@ -29,6 +29,12 @@ function initializeDb() { exports.mochaGlobalSetup = async function () { expect( process.env.NODE_ENV ).to.eq( "test" ); + console.log( "\n\nINITIALIZING TEST ENVIRONMENT\n\n" ); + + if ( !process.env.DB_ALREADY_INITIALIZED ) { + initializeDb( ); + } + // Wait for Postgres console.log( "Waiting for Postgres..." ); await testHelper.waitForPG( 100 ); @@ -37,12 +43,6 @@ exports.mochaGlobalSetup = async function () { console.log( "Waiting for ElasticSearch..." ); await testHelper.waitForES( 100 ); - console.log( "\n\nINITIALIZING TEST ENVIRONMENT\n\n" ); - - if ( !process.env.DB_ALREADY_INITIALIZED ) { - initializeDb( ); - } - console.log( "Creating ES indices" ); await testHelper.createIndices( ); console.log( "Loading ES fixtures" ); From e8adcea7483b5b3fd295b5d0aa71faaacd78f72b Mon Sep 17 00:00:00 2001 From: sylvain-morin Date: Wed, 4 Oct 2023 17:20:32 +0200 Subject: [PATCH 3/4] Run specs with docker compose - add tests related to user email/IP --- test/integration/v1/projects.js | 45 ++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/test/integration/v1/projects.js b/test/integration/v1/projects.js index 03d10e3c..c6c557c1 100644 --- a/test/integration/v1/projects.js +++ b/test/integration/v1/projects.js @@ -302,18 +302,18 @@ describe( "Projects", ( ) => { expect( res.body.per_page ).to.eq( 3 ); expect( res.body.total_results ).to.eq( 3 ); expect( res.body.results.length ).to.eq( 3 ); - expect( res.body.results[0].user.id ).to.eq( 2023092501 ); - expect( res.body.results[0].user ).not.to.be.undefined; - expect( res.body.results[0].user.email ).to.be.undefined; - expect( res.body.results[0].user.last_ip ).to.be.undefined; - expect( res.body.results[1].user.id ).to.eq( 2023092502 ); - expect( res.body.results[1].user ).not.to.be.undefined; - expect( res.body.results[1].user.email ).to.be.undefined; - expect( res.body.results[1].user.last_ip ).to.be.undefined; - expect( res.body.results[2].user.id ).to.eq( 2023092503 ); - expect( res.body.results[2].user ).not.to.be.undefined; - expect( res.body.results[2].user.email ).to.be.undefined; - expect( res.body.results[2].user.last_ip ).to.be.undefined; + const user1 = _.find( res.body.results, u => u.id === 2023092501 ); + expect( user1 ).not.to.be.undefined; + expect( user1.email ).to.be.undefined; + expect( user1.last_ip ).to.be.undefined; + const user2 = _.find( res.body.results, u => u.id === 2023092502 ); + expect( user2 ).not.to.be.undefined; + expect( user2.email ).to.be.undefined; + expect( user2.last_ip ).to.be.undefined; + const user3 = _.find( res.body.results, u => u.id === 2023092503 ); + expect( user3 ).not.to.be.undefined; + expect( user3.email ).to.be.undefined; + expect( user3.last_ip ).to.be.undefined; } ).expect( "Content-Type", /json/ ) .expect( 200, done ); } ); @@ -357,15 +357,18 @@ describe( "Projects", ( ) => { request( this.app ).get( "/v1/projects/2023092501/posts" ) .expect( res => { expect( res.body.total_results ).to.eq( 3 ); - expect( res.body.results[0].user ).not.to.be.undefined; - expect( res.body.results[0].user.email ).to.be.undefined; - expect( res.body.results[0].user.last_ip ).to.be.undefined; - expect( res.body.results[1].user ).not.to.be.undefined; - expect( res.body.results[1].user.email ).to.be.undefined; - expect( res.body.results[1].user.last_ip ).to.be.undefined; - expect( res.body.results[2].user ).not.to.be.undefined; - expect( res.body.results[2].user.email ).to.be.undefined; - expect( res.body.results[2].user.last_ip ).to.be.undefined; + const user1 = _.find( res.body.results, u => u.id === 2023092501 ); + expect( user1 ).not.to.be.undefined; + expect( user1.email ).to.be.undefined; + expect( user1.last_ip ).to.be.undefined; + const user2 = _.find( res.body.results, u => u.id === 2023092502 ); + expect( user2 ).not.to.be.undefined; + expect( user2.email ).to.be.undefined; + expect( user2.last_ip ).to.be.undefined; + const user3 = _.find( res.body.results, u => u.id === 2023092503 ); + expect( user3 ).not.to.be.undefined; + expect( user3.email ).to.be.undefined; + expect( user3.last_ip ).to.be.undefined; } ).expect( "Content-Type", /json/ ) .expect( 200, done ); } ); From 3f4c40755fd29eb1330e1336ee631d6dfd3ce4ca Mon Sep 17 00:00:00 2001 From: sylvain-morin Date: Mon, 16 Oct 2023 15:24:38 +0200 Subject: [PATCH 4/4] Run specs with docker compose --- Dockerfile | 18 +++++++++++++++++- README.md | 23 +++++++++++++++++++++++ docker-compose.test.yml | 8 ++++---- lib/test_helper.js | 12 ++++++++++++ test/hooks.js | 17 +++++++++++------ 5 files changed, 67 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index b7472fd7..bba8ffc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM node:16 +# Platform should be forced to amd64 +# because node-mapnik is not available in arm64 +FROM --platform=linux/amd64 node:16 as base ENV NODE_ENV=development @@ -10,6 +12,20 @@ COPY config_example.js config.js RUN npm install +FROM base as test + +ENV NODE_ENV=test + +RUN apt-get update -qq && apt-get install -y postgresql-client-11 + +COPY . . + +CMD [ "npm", "run", "coverage" ] + +FROM base as development + +ENV NODE_ENV=development + COPY . . EXPOSE 4000 diff --git a/README.md b/README.md index f455fe77..628938df 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,29 @@ Filter by pattern: `NODE_ENV=test ./node_modules/mocha/bin/_mocha --recursive -- You can also add `.only` to a `describe` or `it` call to only run that test when you run `npm test`, e.g. `it.only( "should only run this test" )`. +# Running Tests with Docker + +You can run the tests with Docker Compose. +All required services will be started by the `docker-compose.test.yml` compose file: + +``` +docker compose -f docker-compose.test.yml up -d +``` + +You can follow the tests execution in the logs: + +``` +docker logs -f api-test +``` + +The first time you run the compose file, a local docker image for the API service will be automatically built, from you local GIT checkout. But if later you do some code changes, or update your GIT checkout, you need to re-build the docker image with: + +``` +docker compose -f docker-compose.test.yml build +``` + +This compose build is using the `test` target of the Dockerfile. + # ESLint Please run ESLint to check for syntax formatting errors. To run ESLint, run: `npm run eslint`. Please address any syntax errors before submitting pull requests. ESLint will also run automatically via Github Actions on submitted pull requests along with tests. diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 8879c6a1..da59823f 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -45,7 +45,10 @@ services: api-test: container_name: api-test - build: . + build: + context: . + dockerfile: Dockerfile + target: test environment: NODE_ENV: test INAT_DB_HOST: pg @@ -56,11 +59,8 @@ services: INAT_WEB_HOST: host.docker.internal INAT_DB_NAME: inaturalist_test INAT_ES_INDEX_PREFIX: test - DB_ALREADY_INITIALIZED: true ports: - 4000:4000 - command: - /bin/sh -c "npm install ; npm run coverage" extra_hosts: - "host.docker.internal:host-gateway" diff --git a/lib/test_helper.js b/lib/test_helper.js index 6ef27453..7ddd0dbf 100644 --- a/lib/test_helper.js +++ b/lib/test_helper.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ const fs = require( "fs" ); const _ = require( "lodash" ); const timersPromises = require( "timers/promises" ); @@ -46,6 +47,15 @@ testHelper.waitForES = async numberOfAttempts => { return testHelper.waitForES( numberOfAttempts ); }; +testHelper.closePGConnection = async ( ) => { + await pgClient.connection.end( ); + delete pgClient.connection; +}; + +testHelper.reconnectPGConnection = async ( ) => { + await pgClient.connect( ); +}; + testHelper.forEachIndex = async action => { if ( process.env.NODE_ENV !== "test" ) { return; } const settings = JSON.parse( fs.readFileSync( "schema/settings.js" ) ); @@ -148,6 +158,8 @@ testHelper.testInatJSNoPreload = async ( controller, endpoint, method, done ) => module.exports = { waitForPG: testHelper.waitForPG, waitForES: testHelper.waitForES, + closePGConnection: testHelper.closePGConnection, + reconnectPGConnection: testHelper.reconnectPGConnection, createIndices: testHelper.createIndices, deleteIndices: testHelper.deleteIndices, loadElasticsearchFixtures: testHelper.loadElasticsearchFixtures, diff --git a/test/hooks.js b/test/hooks.js index 039c6392..840f4cf0 100644 --- a/test/hooks.js +++ b/test/hooks.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ const { expect } = require( "chai" ); const { execSync } = require( "child_process" ); const inaturalistjs = require( "inaturalistjs" ); @@ -6,7 +7,7 @@ const testHelper = require( "../lib/test_helper" ); const Taxon = require( "../lib/models/taxon" ); const config = require( "../config" ); -function initializeDb() { +async function initializeDb() { // For tests, we want to make absolutely sure the test database is clean and // new, and we need to make sure that happens before we try to connect to it, // which happens every time we require pg_client, which happens inside of @@ -18,12 +19,18 @@ function initializeDb() { ${config.database.user ? `PGUSER=${config.database.user}` : ""} \ ${config.database.password ? `PGPASSWORD=${config.database.password}` : ""}`; + // Close connection before dropping DB + await testHelper.closePGConnection( ); + console.log( "Dropping existing test database" ); execSync( `${testDbConnectionVar} dropdb --if-exists ${dbname}`, { stdio: [0, 1, 2] } ); console.log( "Creating test database" ); execSync( `${testDbConnectionVar} createdb -O ${config.database.user} ${dbname}`, { stdio: [0, 1, 2] } ); console.log( "Loading test database schema" ); execSync( `${testDbConnectionVar} psql -q -f schema/database.sql -d ${dbname}`, { stdio: [0, 1, 2] } ); + + // Reconnecto to DB + await testHelper.reconnectPGConnection( ); } exports.mochaGlobalSetup = async function () { @@ -31,10 +38,6 @@ exports.mochaGlobalSetup = async function () { console.log( "\n\nINITIALIZING TEST ENVIRONMENT\n\n" ); - if ( !process.env.DB_ALREADY_INITIALIZED ) { - initializeDb( ); - } - // Wait for Postgres console.log( "Waiting for Postgres..." ); await testHelper.waitForPG( 100 ); @@ -43,6 +46,8 @@ exports.mochaGlobalSetup = async function () { console.log( "Waiting for ElasticSearch..." ); await testHelper.waitForES( 100 ); + initializeDb( ); + console.log( "Creating ES indices" ); await testHelper.createIndices( ); console.log( "Loading ES fixtures" ); @@ -57,7 +62,7 @@ exports.mochaGlobalSetup = async function () { exports.mochaHooks = { async beforeAll( ) { - this.timeout( 20000 ); + this.timeout( 60000 ); this.app = await app( ); }, async afterAll( ) {