diff --git a/bin/prod-style-server b/bin/prod-style-server index f1b74b94d0..7369894b13 100755 --- a/bin/prod-style-server +++ b/bin/prod-style-server @@ -25,20 +25,20 @@ fi # Run a production style build (single BE server with static FE) echo "Running yarn install on server..." -docker-compose -f docker-compose.dss.yml run --rm server yarn install --production=false +docker compose -f docker-compose.dss.yml run --rm server yarn install --production=false echo "Running yarn install on frontend..." -docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend install --production=false +docker compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend install --production=false echo "Building backend..." -docker-compose -f docker-compose.dss.yml run --rm server yarn build +docker compose -f docker-compose.dss.yml run --rm server yarn build echo "Building frontend..." -docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend run build +docker compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend run build # Starting services echo "Starting services..." -docker-compose -f docker-compose.dss.yml up -d +docker compose -f docker-compose.dss.yml up -d # Running database migrations echo "Running database migrations..." -docker-compose -f docker-compose.dss.yml run --rm server yarn db:migrate:ci +docker compose -f docker-compose.dss.yml run --rm server yarn db:migrate:ci echo "Server setup completed successfully." diff --git a/bin/run-tests b/bin/run-tests index 8b510350f1..c722780ab9 100755 --- a/bin/run-tests +++ b/bin/run-tests @@ -9,7 +9,7 @@ log() { check_exit() { if [[ "$1" -ne 0 ]]; then - echo "$lr last docker-compose command failed" + echo "$lr last docker compose command failed" ((exit_code++)) fi } @@ -42,19 +42,19 @@ main() { esac done - docker-compose \ + docker compose \ -f 'docker-compose.test.yml' \ down --volumes --remove-orphans log "Running tests in using test config 'docker-compose.test.yml'" # Start containers if [[ $docker == "true" ]]; then - docker-compose -f 'docker-compose.test.yml' up -d + docker compose -f 'docker-compose.test.yml' up -d check_exit "$?" else - docker-compose -f 'docker-compose.test.yml' up -d test-db + docker compose -f 'docker-compose.test.yml' up -d test-db check_exit "$?" - IFS=: read host port <<< "$(docker-compose -f 'docker-compose.test.yml' port test-db 5432)" + IFS=: read host port <<< "$(docker compose -f 'docker-compose.test.yml' port test-db 5432)" echo "Postgres is running on port $port" export POSTGRES_PORT=$port fi @@ -177,7 +177,7 @@ main() { # Cleanup echo log "Cleaning up test containers" - docker-compose \ + docker compose \ -f 'docker-compose.test.yml' \ down --volumes --remove-orphans check_exit "$?" diff --git a/bin/test-backend-ci b/bin/test-backend-ci index f7ead1634d..12049143b3 100755 --- a/bin/test-backend-ci +++ b/bin/test-backend-ci @@ -4,7 +4,7 @@ declare -i exit_code=0 check_exit() { if [[ "$1" -ne 0 ]]; then - echo "$lr last docker-compose command failed" + echo "$lr last docker compose command failed" ((exit_code++)) fi } diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index f66e3579d5..96f40bc877 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -1,4 +1,3 @@ -version: "3.7" services: backend: build: diff --git a/docker-compose.dss.yml b/docker-compose.dss.yml index eff6c8809f..b9bbdb18a0 100644 --- a/docker-compose.dss.yml +++ b/docker-compose.dss.yml @@ -1,4 +1,3 @@ -version: "3.2" services: server: build: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index a96aebee71..0a529a9a5d 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,8 +1,9 @@ -version: "3.7" services: backend: build: context: . + profiles: + - minimal_required command: yarn server user: ${CURRENT_USER:-root} ports: @@ -31,6 +32,8 @@ services: frontend: build: context: . + profiles: + - minimal_required command: yarn start user: ${CURRENT_USER:-root} stdin_open: true @@ -46,6 +49,8 @@ services: worker: build: context: . + profiles: + - minimal_required command: yarn worker env_file: .env depends_on: @@ -59,6 +64,8 @@ services: - ".:/app:rw" owasp_zap_backend: image: softwaresecurityproject/zap-stable:latest + profiles: + - full_stack platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report_.html @@ -69,6 +76,8 @@ services: - backend owasp_zap_similarity: image: softwaresecurityproject/zap-stable:latest + profiles: + - full_stack platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 0410c967f4..d3f92070bc 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,4 +1,3 @@ -version: "3.5" services: test-backend: build: diff --git a/docker-compose.yml b/docker-compose.yml index 5fa3c630ad..7603af4423 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,8 @@ -version: "3.7" services: api-docs: image: redocly/redoc + profiles: + - full_stack ports: - "5003:80" volumes: @@ -12,6 +13,8 @@ services: image: postgres:15.6 container_name: postgres_docker env_file: .env + profiles: + - minimal_required ports: - "5432:5432" volumes: @@ -19,6 +22,8 @@ services: shm_size: 1g minio: image: minio/minio:RELEASE.2024-01-01T16-36-33Z + profiles: + - full_stack env_file: .env ports: - "9000:9000" @@ -28,12 +33,16 @@ services: command: server /data --console-address ":9001" aws-cli: image: amazon/aws-cli + profiles: + - full_stack env_file: .env command: ["--endpoint-url", "http://minio:9000", "s3api", "create-bucket", "--bucket", "$S3_BUCKET"] depends_on: - minio clamav-rest: image: ajilaag/clamav-rest + profiles: + - full_stack ports: - "9443:9443" environment: @@ -41,6 +50,8 @@ services: similarity_api: build: context: ./similarity_api + profiles: + - minimal_required ports: - "9100:8080" env_file: .env @@ -50,18 +61,24 @@ services: - "./similarity_api/src:/app:rw" redis: image: redis:5.0.6-alpine + profiles: + - minimal_required command: ['redis-server', '--requirepass', '$REDIS_PASS'] env_file: .env ports: - "6379:6379" mailcatcher: image: schickling/mailcatcher + profiles: + - full_stack ports: - "1025:1025" - "1080:1080" testingonly: build: context: . + profiles: + - full_stack ports: - "9999:9999" depends_on: @@ -74,6 +91,8 @@ services: - NODE_ENV=development sftp: image: jmcombs/sftp:alpine + profiles: + - full_stack volumes: - ./test-sftp:/home/tta_ro/ProdTTAHome - ./test-sftp/sshd_config:/etc/ssh/sshd_config diff --git a/docs/testing.md b/docs/testing.md index 5268d7fe57..2e5022f7f5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -38,11 +38,11 @@ By default, `./bin/run-tests` will run both backend and frontend tests. If you w When running tests in Docker, be aware that there are tests that will modify/delete database records. For tests to run, the 'db' service needs to exist and `db:migrate` and `db:seed` need to have been run (to create the tables and populate certain records). -In the `docker-compose.yml` configuration, the database is set up to persist to a volume, "dbdata", so database records will persist between runs of the 'db' service, unless you remove that volume explicitly (e.g. `docker volume rm` or `docker-compose down --volumes`). +In the `docker-compose.yml` configuration, the database is set up to persist to a volume, "dbdata", so database records will persist between runs of the 'db' service, unless you remove that volume explicitly (e.g. `docker volume rm` or `docker compose down --volumes`). -### Notes on docker-compose and multiple configurations +### Notes on docker compose and multiple configurations -`docker-compose` has a feature for providing multiple `docker-compose.*.yml` files where subsequent files can override settings in previous files, which sounds like it would suit the use case of running docker for local development and for testing. However, the ability to [override configurations](https://docs.docker.com/compose/extends/#adding-and-overriding-configuration) is limited. While experimenting with overrides, it became clear that doing so would require a minimum of three docker-compose.yml files: one "base", one for local development, one for running tests. Trying to compose docker-compose.yml files would be complicated. +`docker compose` has a feature for providing multiple `docker-compose.*.yml` files where subsequent files can override settings in previous files, which sounds like it would suit the use case of running docker for local development and for testing. However, the ability to [override configurations](https://docs.docker.com/compose/extends/#adding-and-overriding-configuration) is limited. While experimenting with overrides, it became clear that doing so would require a minimum of three docker-compose.yml files: one "base", one for local development, one for running tests. Trying to compose docker-compose.yml files would be complicated. In addition, while experimenting with multiple configuration files, it became clear that docker was unable to differentiate between different versions of the same service. Trying to override the 'db' service for testing would not work as expected: if the local/dev 'db' service had already been created, that one would be used when tests were run. diff --git a/package.json b/package.json index facdc83bf6..32342234c9 100644 --- a/package.json +++ b/package.json @@ -52,29 +52,31 @@ "db:seed:prod": "node_modules/.bin/sequelize db:seed:all --options-path .production.sequelizerc", "db:seed:undo": "node_modules/.bin/sequelize db:seed:undo:all", "db:seed:undo:prod": "node_modules/.bin/sequelize db:seed:undo:all --options-path .production.sequelizerc", - "docker:deps": "docker-compose run --rm backend yarn install && docker-compose run --rm frontend yarn install", + "docker:deps": "docker compose run --rm backend yarn install && docker compose run --rm frontend yarn install && docker compose run --rm worker yarn install", "docker:reset": "./bin/reset-all", - "docker:start": "docker-compose up", - "docker:start:debug": "docker-compose --compatibility -f docker-compose.yml -f docker-compose.debug.yml up", - "docker:stop": "docker-compose down", - "docker:dbs:start": "docker-compose -f 'docker-compose.yml' up", - "docker:dbs:stop": "docker-compose -f 'docker-compose.yml' down", + "docker:start": "docker compose --profile minimal_required up", + "docker:start:full": "docker compose --profile full_stack --profile minimal_required up", + "docker:stop:full": "docker compose --profile full_stack --profile minimal_required down", + "docker:start:debug": "docker compose --compatibility -f docker-compose.yml -f docker-compose.debug.yml up", + "docker:stop": "docker compose --profile minimal_required down", + "docker:dbs:start": "docker compose -f 'docker-compose.yml' up", + "docker:dbs:stop": "docker compose -f 'docker-compose.yml' down", "docker:test": "./bin/run-tests", - "docker:test:be": "docker-compose run --rm backend yarn test", - "docker:lint": "docker-compose run --rm backend yarn lint:ci && docker-compose run --rm frontend yarn lint:ci", - "docker:lint:fix": "docker-compose run --rm backend yarn lint:fix && docker-compose run --rm frontend yarn lint:fix", - "docker:shell:frontend": "docker-compose run --rm frontend /bin/bash", - "docker:shell:backend": "docker-compose run --rm backend /bin/bash", - "docker:db:migrate": "docker-compose run --rm backend node_modules/.bin/sequelize db:migrate && yarn docker:ldm", - "docker:db:migrate:undo": "docker-compose run --rm backend node_modules/.bin/sequelize db:migrate:undo", - "docker:db:seed": "docker-compose run --rm backend yarn db:seed", - "docker:db:seed:undo": "docker-compose run --rm backend yarn db:seed:undo", - "docker:import:system": "docker-compose run --rm backend yarn import:system", - "docker:ldm": "docker-compose run --rm backend yarn ldm", - "docker:makecolors": "docker-compose run --rm frontend yarn makecolors", - "docker:yarn": "docker-compose run yarn", - "docker:yarn:fe": "docker-compose run --rm frontend yarn", - "docker:yarn:be": "docker-compose run --rm backend yarn", + "docker:test:be": "docker compose run --rm backend yarn test", + "docker:lint": "docker compose run --rm backend yarn lint:ci && docker compose run --rm frontend yarn lint:ci", + "docker:lint:fix": "docker compose run --rm backend yarn lint:fix && docker compose run --rm frontend yarn lint:fix", + "docker:shell:frontend": "docker compose run --rm frontend /bin/bash", + "docker:shell:backend": "docker compose run --rm backend /bin/bash", + "docker:db:migrate": "docker compose run --rm backend node_modules/.bin/sequelize db:migrate && yarn docker:ldm", + "docker:db:migrate:undo": "docker compose run --rm backend node_modules/.bin/sequelize db:migrate:undo", + "docker:db:seed": "docker compose run --rm backend yarn db:seed", + "docker:db:seed:undo": "docker compose run --rm backend yarn db:seed:undo", + "docker:import:system": "docker compose run --rm backend yarn import:system", + "docker:ldm": "docker compose run --rm backend yarn ldm", + "docker:makecolors": "docker compose run --rm frontend yarn makecolors", + "docker:yarn": "docker compose run yarn", + "docker:yarn:fe": "docker compose run --rm frontend yarn", + "docker:yarn:be": "docker compose run --rm backend yarn", "import:reports:local": "./node_modules/.bin/babel-node ./src/tools/importSSActivityReports.js", "import:reports": "node ./build/server/src/tools/importSSActivityReports.js", "import:goals:local": "./node_modules/.bin/babel-node ./src/tools/importTTAPlanGoals.js",