diff --git a/.github/workflows/api-tests.yaml b/.github/workflows/api-tests.yaml index 397c376e..6ca8e066 100644 --- a/.github/workflows/api-tests.yaml +++ b/.github/workflows/api-tests.yaml @@ -23,7 +23,7 @@ jobs: - name: Create env-files env: GEONAMES_USERNAME: ${{ secrets.GEONAMES_USERNAME }} - run: GEONAMES_USERNAME=$GEONAMES_USERNAME npm run config:all + run: GEONAMES_USERNAME=$GEONAMES_USERNAME npm run config - name: 'Start containers' run: npm run start:anon:api -- --wait -d diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index dd65d988..0521f7bd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -23,7 +23,7 @@ jobs: - name: Create env-files env: GEONAMES_USERNAME: ${{ secrets.GEONAMES_USERNAME }} - run: GEONAMES_USERNAME=$GEONAMES_USERNAME npm run config:all + run: GEONAMES_USERNAME=$GEONAMES_USERNAME npm run config - name: Install root dependencies run: npm ci diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 5e6410b8..8863a303 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -15,8 +15,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Copy .anon.env - run: cp .template.env .anon.env + - name: Create env-files + env: + GEONAMES_USERNAME: ${{ secrets.GEONAMES_USERNAME }} + run: GEONAMES_USERNAME=$GEONAMES_USERNAME npm run config - name: Setup node uses: actions/setup-node@v4 @@ -26,5 +28,8 @@ jobs: - name: 'Install dependencies' run: npm run setup + - name: Run backend unit-tests + run: npm run test:unit:backend + - name: Run frontend unit-tests run: npm run test:unit:frontend diff --git a/README.md b/README.md index 37e2b02f..761467ff 100644 --- a/README.md +++ b/README.md @@ -13,80 +13,23 @@ The new version is not deployed yet. **Basic commands** :computer: - `npm run setup` - - Installs all node modules, and generates prisma client + - Creates all files and directories required to run the project - `npm run dev` - - Uses `docker-compose.dev.yml` - - Run in development mode: with hot reload - - To create test users visit `http://localhost:4000/test/create-test-users` - -Testing: - -- `npm run check` - - Runs `tsc` and `eslint` for frontend and backend: Use this to check for errors before committing. + - Runs the application in development mode + - Supports hot reloading + - Serves frontend by default at `http://localhost:5173` - `npm run test:api` - - Runs the api-tests in a docker container - - Requires database to be running. Can be achieved by `npm run start:anon -- db-anon` or just `npm run start:anon:api`. - - Uses `test_data` - - Uses `.test.env` - - `npm run test:api:build` builds the container used to run the tests. - - `npm run test:api:run` runs the previously built container. + - Runs api-tests inside Docker + - Requires database to be running (start one with correct data: `npm run start:anon:api`) - `npm run test:e2e` - - Runs e2e-tests inside a docker container. - - Requires `frontend`, `backend` and `database` to be running. Can be achieved by running `npm run start:anon`. - - Uses `test_data` -- `npm run test:e2e:local` - - Runs cypress-tests locally (not in docker) in headless mode. - - Requires `npm run start:anon` to be running first. -- `npm run test:api:local` - - Runs backend api tests locally (not in docker). - - Requires `db-anon` to be running first. (Note: backend doesn't need to be running.) - - Requires `.test.env` -- `npm run cypress` - - Opens cypress GUI. You can run tests and see what they do. - -Windows testing: - -- Both `e2e` and `api-tests` have a Windows specific command. - - `e2e` is `npm run test:e2e:windows` - - `api-tests` is `npm run test:api:windows` - - For `api-tests`: inside `.test.env`, you need to change all instances of `localhost` to `host.docker.internal`. - - These include `MARIADB_HOST`, `DATABASE_URL` and `LOG_DATABASE_URL`. - - Note: these are not tested yet. - -Test env-file: - -- `.test.env` - - `api-tests` requires `.test.env` to set the correct environment variables. - - This is created by copying `.anon.env` and changing all instances of `nowdb-db-anon` to `localhost`. - - These include `MARIADB_HOST`, `DATABASE_URL` and `LOG_DATABASE_URL`. - -Coverage: - -- Coverage opening - - Coverage can be opened by opening the `coverage/lcov-report/index.html` with your browser of choice. -- `coverage:sed` - - Fixes issues with `.nyc_output` paths. Replaces the paths inside Docker to `$PWD` (current directory). -- `coverage` - - First fixes paths with `coverage:sed` and then recreates `coverage/lcov-report` to match the new path. -- `coverage:report` - - Alias to `npm run coverage`. Used by cypress e2e tests so that `coverage:sed` gets executed automatically. - -Run with the same image that is used in production: - -- `npm start` - - Uses `docker-compose.yml` - - This builds the code and runs it. Mostly you'll want to use this after major technical changes to check the code works also in production mode, before deploying. And then you'll likely want to do: `npm start -- --build` to force rebuild of containers. - -Docker: - -- `npm run dev:down` - - Stops and removes the dev container and it's volumes. -- `-- --build` - - Useful flag to force docker to rebuild the containers. Example usage for dev: `npm run dev -- --build` + - Runs api-tests inside Docker + - Requires frontend, backend and database to be running (start all with `npm run start:anon`) **Further documentation** -:rocket: [Initialize app & restore db](documentation/init.md) How to get the app up & running, and initialize database from sql-dumps +:rocket: [Setup app & restore db](documentation/setup.md) How to get the app up & running, and initialize database from sql-dumps + +:keyboard: [List of all commands](documentation/commands.md) A comprehensive list of all of the most important commands :wrench: [Technical explanations](documentation/guides/technical_explanations.md) Explanations of how things work, and guides for some more rare tasks like updating database-schema diff --git a/backend/Dockerfile b/backend/Dockerfile index 6f200afb..bb87f059 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -49,16 +49,3 @@ EXPOSE 4000 ENV NODE_ENV=production CMD ["node", "src/index.js"] - - -FROM node:22-alpine3.20 AS api-tests - -USER node -COPY --chown=node --from=backend-build /usr/src/app /usr/src/app - -WORKDIR /usr/src/app/backend - -RUN npm ci -RUN npm run prisma - -CMD ["npm", "run", "test:api"] diff --git a/backend/dev.Dockerfile b/backend/dev.Dockerfile index e5ad361f..5b241bde 100644 --- a/backend/dev.Dockerfile +++ b/backend/dev.Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-alpine3.20 +FROM node:22-alpine3.20 as dev ENV TZ="Europe/Helsinki" @@ -10,6 +10,10 @@ WORKDIR /usr/src/app RUN mkdir -p /usr/src/app/frontend/src/shared/validators +COPY --chown=node ./frontend/src/backendTypes.d.ts /usr/src/app/frontend/src/backendTypes.d.ts +COPY --chown=node ./frontend/src/types.ts /usr/src/app/frontend/src/types.ts +COPY --chown=node ./frontend/src/validators /usr/src/app/frontend/src/validators + COPY --chown=node ./backend/package.json backend/ COPY --chown=node ./backend/package-lock.json backend/ COPY --chown=node ./backend/prisma/schema.prisma backend @@ -26,3 +30,8 @@ COPY --chown=node ./backend . RUN npm run prisma CMD ["npm", "run", "dev"] + + +FROM dev as api-tests + +CMD ["npm", "run", "test:api"] diff --git a/backend/package.json b/backend/package.json index 034959b5..42291f16 100644 --- a/backend/package.json +++ b/backend/package.json @@ -12,7 +12,7 @@ "start": "node build/index.js", "lint": "eslint --report-unused-disable-directives --max-warnings 0", "prisma": "prisma generate --schema prisma/schema.prisma && prisma generate --schema prisma/schema_log.prisma", - "clean": "rm -rf node_modules && rm -rf prisma/generated && rm -rf build" + "clean": "rm -rf node_modules && rm -rf prisma/generated && rm -rf build && rm -rf coverage" }, "author": "", "devDependencies": { diff --git a/docker-compose.anon.yml b/docker-compose.anon.yml index a8f28d5b..30b1fe09 100644 --- a/docker-compose.anon.yml +++ b/docker-compose.anon.yml @@ -22,6 +22,7 @@ services: build: context: . dockerfile: ./backend/dev.Dockerfile + target: dev volumes: - ./backend/src:/usr/src/app/backend/src - ./frontend/src/:/usr/src/app/frontend/src/ diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0d19359d..9a103211 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -16,6 +16,7 @@ services: build: context: . dockerfile: ./backend/dev.Dockerfile + target: dev volumes: - ./backend/src:/usr/src/app/backend/src - ./frontend/src/:/usr/src/app/frontend/src/ @@ -30,7 +31,7 @@ services: env_file: .dev.env volumes: - ./data/:/docker-entrypoint-initdb.d - - nowdb-db:/var/lib/mysql/data + - nowdb-db-dev:/var/lib/mysql/data ports: - 127.0.0.1:3306:3306 @@ -46,4 +47,4 @@ services: - "9090:443" volumes: - nowdb-db: + nowdb-db-dev: diff --git a/documentation/commands.md b/documentation/commands.md new file mode 100644 index 00000000..14e9d67c --- /dev/null +++ b/documentation/commands.md @@ -0,0 +1,91 @@ +# npm commands + +## Initialization +* `setup` + * Installs node_modules to project root, backend and frontend + * Sets up Prisma in backend + * Creates all config files +* `config:all` + * Creates all config files: `.dev.env`, `.anon.env` and `.test.env` + +## Running the application +* `start` + * Runs the application in the production environment + * Opens backend, frontend and a MariaDB database (initialized from `data` directory) + * Utilizes `.env` +* `start:anon` + * Runs the application in dev environment + * Opens backend, frontend, phpMyAdmin and a MariaDB database (initialized from `test_data` directory) + * Utilizes `.anon.env` +* `dev` + * Runs the application in development environment + * Enables hotswap (docker compose doesn't need to be restarted on changes inside `backend/src` or `frontend/src`) + * Note: When editing for example `package.json` the docker needs to be manually restarted. + * Opens backend, frontend, phpMyAdmin, and a MariaDB database (initialized from `data` directory) + * Utilizes `.dev.env` + * To create test users visit `http://localhost:4000/test/create-test-users` + +## Testing +* `test:api` + * Runs tests for backend API + * Tests are located in `backend/src/api-tests` + * Tests are executed inside Docker + * Requires a database running with (preferrably, see last point) `test_data` + * Utilizes `.test.env` + * Anon version of the database is recommended be used by these tests + * Resets the database with `test_data` multiple times! +* `test:api:local` + * Same as `test:api` but tests are executed locally +* `test:e2e` + * Runs e2e tests with cypress + * Tests are located in `cypress/e2e` + * Tests are executed inside Docker + * Requires backend, frontend and database (with preferably `test_data`, see last point) to be running + * Expects to find frontend at `` defined in `cypress/config.js` + * Expects to find database reset API-point at `` defined in `cypress.config.js` + * Both can be also defined as a environment variable + * Resets the database with `test_data` multiple times! +* `test:e2e:local` + * Same as `test:e2e` but tests are executed locally + * Change the browser by running `npm run test:e2e:local -- --browser ` +* `test:unit` + * Runs all unit tests defined in `backend/src/unit-tests` and `frontend/src/tests` + * Doesn't require containers running + * Executed locally +* `test:{api,e2e}:windows` + * Executes api or e2e command with (supposed) Windows support + * Not actually tested on Windows: "should work" + * For `api-tests` to work with Windows: inside `.test.env`, you need to change all instances of `localhost` to `host.docker.internal`. + * These include `MARIADB_HOST`, `DATABASE_URL` and `LOG_DATABASE_URL`. + * A short script that should do the same: `sed -i -e s|localhost|host.docker.internal|g .test.env` + + +## Utilities +* `check` + * Runs linting and typescript checking for backend, frontend and cypress +* `anon:down` + * Closes all docker containers running anon versions + * REMOVES `nowdb-db-anon` VOLUME PERMANENTLY + * This is usually not a problem as the normal development environment uses a different volume +* `dev:down` + * Closes all docker containers running dev versions + * REMOVES `nowdb-db-dev` VOLUME PERMANENTLY +* `clean` + * Removes all node_modules, build directories, prisma's autogenerated files, .nyc_output and coverage directories +* `coverage` + * Creates coverage report based on `.nyc_output` + * Replaces the incorrect paths of the files from the path inside Docker container to the current working directory (`$PWD`) + * Automatically run by `e2e` each time +* `coverage:report` + * Alias to `coverage` for cypress to create coverage while running tests + * https://github.com/cypress-io/code-coverage?tab=readme-ov-file#custom-nyc-command + + +### Be aware +* Weird problems with docker? + * Add `-- --build` after for example `npm run start:anon` to force rebuilding all containers + * Sometimes docker doesn't rebuild containers when needed and it leads to hard-to-find bugs + +* Difference between `cmd1 && cmd2` and `cmd1; cmd2` + * `&&`: `cmd2` is executed only when `cmd1` returns 0 (=is succesful) + * `;`: `cmd2` is always executed (after `cmd1`) diff --git a/documentation/guides/technical_explanations.md b/documentation/guides/technical_explanations.md index ab1099ef..e070efd8 100644 --- a/documentation/guides/technical_explanations.md +++ b/documentation/guides/technical_explanations.md @@ -1,4 +1,3 @@ - ### Database **DB-dump restoring process** @@ -41,3 +40,94 @@ Sometimes you may want to add some test user or entries into the anonymous data **Mariadb-connector and transactions** Prisma does not have proper support for connecting to multiple databases, and transactionality between them is therefore impossible. We need transactional write to both now and now_log databases. Therefore we use mariadb-connector for the write operations that need to be logged into the log-database. + + +### Docker + +#### Dockerfile + +There are two variants of Dockerfile used for both backend and frontend: + +1. Production (`Dockerfile`) + * Installs only packages required in production (no devDependencies) + * Copies only required files to the image + * Tries to be minimal so the image would be fast and small +2. Development (`dev.Dockerfile`) + * Installs all packages from `package.json` + * Copies all files from `backend` to the image (not counting `.dockerignore`d files) + * Has multiple stages (dev and api-tests) + * Learn more about multi-stage Dockerfiles: https://docs.docker.com/build/building/multi-stage/ + * Includes everything that is required / helps with development and testing + + +#### Docker compose + +There are three variants of docker compose used in this project. + +1. Production (`docker-compose.yml`) + * Uses `{backend,frontend}/Dockerfile` for both backend and frontend + * Uses `.env` for environment variables + * Uses `data` for database initialization + * Doesn't start phpMyAdmin + * Can be started with `npm run start` +2. Dev (`docker-compose.dev.yml`) + * Uses `{backend,frontend}/dev.Dockerfile` + * Uses `.dev.env` + * Uses `data` for database initialization + * Can be started with `npm run dev` +3. Anon (`docker-compose.anon.yml`) + * Uses `{backend,frontend}/dev.Dockerfile` + * Uses `.anon.env` + * Uses `test_data` for database initialization + * Can be started with `npm run start:anon` + + +### Openshift + +Staging and production is deployed in Openshift. You should read more about Openshift's own documentation +and Kubernetes documentation to understand more. + +#### Updating deployments + +Auto-deploying pods when new images are pulled from Github doesn't work at the moment because of +a bug in Openshift. Until it is fixed the deployments have to be updated manually. + +1. Select role as developer if there is an option. Otherwise skip this. +2. Go to `Topology` +3. Select backend or frontend +4. From `Actions` dropdown menu select "`Edit Deployment`" +5. Select `Form view` if not selected. +6. Tick `Auto deploy when new Image is available` +7. Save + +If there was a new version of an image, a new pod should be created and the old one destroyed after +the new pod is up and running. + +Same goes for production. + +Trying to update the staging right after merging a pull request might not always work because +Openshift pulls the new image from Github packages roughly every 15mins. + + + +### Github actions + +This project utilizes Github actions for linting, testing and building the project. At the moment +pull requests cannot be merged if the all of the pull-request-actions do not pass. + +The following actions are run on every pull request: +* `lint.yaml` + * Lints the whole project and fails if there is linting issues +* `api-tests.yaml` + * Runs api tests inside Docker, passes if all tests pass +* `e2e.yaml` + * Runs e2e tests inside Docker, passes if all tests pass +* `unit-tests.yaml` + * Runs all unit tests, passes if all tests pass + +And the following are run on only merges to main: +* `build.yaml` + * Builds staging version of both backend and frontend and creates Github package out of + both of them. If the triggering event happens to be a release, a production package is made. +* `phpmyadmin.yaml` + * Builds the patched phpMyAdmin image and creates a Github package diff --git a/documentation/init.md b/documentation/setup.md similarity index 82% rename from documentation/init.md rename to documentation/setup.md index 59eae4e0..b3974578 100644 --- a/documentation/init.md +++ b/documentation/setup.md @@ -10,7 +10,7 @@ See [technical explanations](./guides/technical_explanations.md) for how this wo **Init the project** :rocket: -Requirements: Some modern installation of Docker engine with Docker compose. +Requirements: npm and some modern installation of Docker engine with Docker compose. 1. `npm run setup` in project root to install all node modules, generate prisma-client and create all required config files (doesn't overwrite previous ones!). 2. If you wish to use the staging database, you need to copy the dump files into `data/sqlfiles/` directory. See example of how anonymized test data is in `test_data/sqlfiles/` @@ -19,6 +19,16 @@ Requirements: Some modern installation of Docker engine with Docker compose. **Errors while setting up** +Before any other debugging steps: +1. Make sure main branch is up to date +2. Make sure again, you didn't actually check did you... +3. Re-setup your environment with `npm run setup` +4. Remove existing containers and volumes `npm run :down` +5. Force rebuild it with `-- --build` flag: `npm run -- --build` +6. If problems persist, scream for help and hope someone hears you +7. Continue with other debugging steps +8. Repeat + ***Database*** If the database fails to start with an error message `Cannot open directory /docker-entrypoint-initdb.d/. Permission denied`, the problem is most likely permissions. `dev` uses `data` directory and `anon` + `test` uses `test_data`. Both directories and the files inside need the permissions `read` and `execute`. In Linux/GNU this can be achieved by running `chmod 755 DIRECTORY -R` where DIRECTORY is the wanted directory, `test_data` or `data`. @@ -31,7 +41,7 @@ Both directories and the files inside need the permissions `read` and `execute`. - The command gives permissions `read` and `execute` for directories and `read` for files inside `backend/src`. - NOTE: This gives `write` for all files and directories for current user. You can disable this by changing the first number to match the others. -* `Cannot find module ../../../frontend/src/types` +* `Cannot find module ../../../frontend/src/shared/types` - Permission error with `frontend/src` - Fixed by running `find frontend/src -type d -exec chmod 755 {} \; && find frontend/src -type f -exec chmod 644 {} \;` (on Linux). - The command gives permissions `read` and `execute` for directories and `read` for files inside `frontend/src`. diff --git a/package.json b/package.json index 9068d67c..80cd2768 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "NOW - New and Old Worlds - Database of fossil mammals", "main": "index.js", "scripts": { - "setup": "npm run install:all && npm run prisma && npm run config:all", + "setup": "npm run install:all && npm run prisma && npm run config", "install:all": "npm i && cd backend && npm i && cd ../frontend && npm i && cd ..", "start": "docker compose -f docker-compose.yml up", "start:anon": "docker compose -f docker-compose.anon.yml up", @@ -17,13 +17,9 @@ "check:frontend": "echo \"Running frontend tsc ...\" && cd frontend && npx tsc --noEmit && echo \"Running frontend eslint ...\" && npm run lint", "check:cypress": "echo \"Running cypress eslint ...\" && npm run lint:cypress", "lint:cypress": "eslint --report-unused-disable-directives --max-warnings 0 cypress/e2e/**.cy.js", - "test:up": "docker compose -f docker-compose.test.yml up db-anon backend frontend", - "test:down": "docker compose -f docker-compose.test.yml down -v", - "test:ci": "docker compose -f docker-compose.test.yml up", - "test:backend": "npm run test:unit && npm run test:api", - "test:unit": "cd backend && npm run test:unit", + "test:backend": "npm run test:unit:backend && npm run test:api", "test:api": "npm run test:api:build && npm run test:api:run", - "test:api:build": "docker build -t nowdb-api-tests . -f backend/Dockerfile --target api-tests", + "test:api:build": "docker build -t nowdb-api-tests . -f backend/dev.Dockerfile --target api-tests", "test:api:run": "npm run test:api:run:base -- -it nowdb-api-tests", "test:api:run:base": "docker run --rm --network=host --user node --env-file .test.env --volume ./test_data:/usr/src/app/test_data", "test:api:windows": "npm run test:api:build && npm run test:api:run:windows", @@ -34,6 +30,8 @@ "test:e2e:windows": "npm run test:e2e:base:windows -- -it --entrypoint cypress cypress/included:13.15.0 run --browser chrome", "test:e2e:base:windows": "docker run --user node -v $PWD:/e2e -w /e2e --env CYPRESS_baseUrl=http://host.docker.internal:5173 --env CYPRESS_databaseResetUrl=http://host.docker.internal:4000/test/reset-test-database", "test:e2e:local": "cypress run --browser chrome", + "test:unit": "npm run test:unit:backend && npm run test:unit:frontend", + "test:unit:backend": "cd backend && npm run test:unit", "test:unit:frontend": "cd frontend && npm run test", "test:ci:api": "npm run test:api:build && npm run test:api:run:base -- nowdb-api-tests", "test:ci:e2e": "npm run test:e2e:base -- --user 1001:1001 --entrypoint cypress cypress/included:13.15.0 run --browser chrome", @@ -43,8 +41,8 @@ "cypress": "cypress open", "prisma": "cd backend && npx prisma generate --schema prisma/schema.prisma && npx prisma generate --schema prisma/schema_log.prisma", "reset-anon-db": "docker exec -it nowdb-db-anon sh -c \"mariadb --user root -padmin now_test < /docker-entrypoint-initdb.d/sqlfiles/now_test.sql\" && docker exec -it nowdb-db-anon sh -c \"mariadb --user root -padmin now_log_test < /docker-entrypoint-initdb.d/sqlfiles/now_log_test.sql\"", - "clean:all": "rm -rf node_modules && cd backend && npm run clean && cd ../frontend && npm run clean", - "config:all": "npm run config:dev && npm run config:anon && npm run config:test", + "clean": "rm -rf node_modules && rm -rf coverage & rm -rf .nyc_output && rm -rf cypress/screenshots && rm -rf cypress/downloads && cd backend && npm run clean && cd ../frontend && npm run clean", + "config": "npm run config:dev && npm run config:anon && npm run config:test", "config:test": "npm run config:test:base && npm run config:test:host && npm run config:test:coordinator && npm run config:geonames -- .test.env && npm run config:test:yaml && npm run config:test:quotes", "config:test:base": "cp -n .template.env .test.env", "config:test:host": "sed -i -e \"s|nowdb-db-dev|localhost|g\" .test.env",