From ea601c9c313d3980c22482509d97876fcb33c6a2 Mon Sep 17 00:00:00 2001 From: Heorhii Date: Fri, 5 May 2023 14:28:44 +0200 Subject: [PATCH 1/6] add redis config --- application/config/redis.js | 9 +++++++++ application/db/redis/start.js | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 application/config/redis.js diff --git a/application/config/redis.js b/application/config/redis.js new file mode 100644 index 00000000..36afbbbd --- /dev/null +++ b/application/config/redis.js @@ -0,0 +1,9 @@ +({ + apiRedis: { + socket: { + host: process.env.REDIS_HOST ?? '127.0.0.1', + port: process.env.REDIS_PORT ?? 6379, + }, + password: process.env.PASS, + }, +}); diff --git a/application/db/redis/start.js b/application/db/redis/start.js index 6c945102..30433c28 100644 --- a/application/db/redis/start.js +++ b/application/db/redis/start.js @@ -2,7 +2,7 @@ async () => { if (application.worker.id === 'W1') { console.debug('Connect to redis'); } - const client = npm.redis.createClient(); + const client = npm.redis.createClient(config.redis.apiRedis); db.redis.client = client; client.on('error', () => { if (application.worker.id === 'W1') { From 0001680e23f1141d130b808d8e5cd6a4b731b62d Mon Sep 17 00:00:00 2001 From: Heorhii Date: Thu, 4 May 2023 21:22:56 +0200 Subject: [PATCH 2/6] Update test.yml --- .github/workflows/test.yml | 89 +++++++++++++++++++++++++++++++++++ .prettierignore | 1 + Dockerfile | 2 +- application/db/redis/start.js | 2 +- docker-compose.yml | 12 ++++- test-docker-compose.yml | 37 +++++++++++++++ 6 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 test-docker-compose.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51c1d962..0de20eaa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,17 +18,106 @@ jobs: steps: - uses: actions/checkout@v3 + + - name: Install PostgreSQL and Redis (on MacOS) + if: runner.os == 'macOS' + run: | + # brew install postgresql + brew install redis + brew services start postgresql + brew services start redis + + - name: Check services status (on MacOS) + if: runner.os == 'macOS' + run: | + # until brew services list | grep -E 'postgresql[[:space:]]+started' && brew services list | grep -E 'redis[[:space:]]+started'; do sleep 5; done + until pg_isready -h localhost -p 5432 && redis-cli ping; do sleep 5; done + + - name: Set up PostgreSQL (on MacOS) + if: runner.os == 'macOS' + run: | + createuser -s postgres + cd database + bash setup.sh + + - name: Setup pg (on Windows) + if: runner.os == 'Windows' + run: | + echo "unix_socket_directories = ''" >> "$PGDATA/postgresql.conf" + echo "port = 5432" >> "$PGDATA/postgresql.conf" + $PG_BIN = "C:\Program Files\PostgreSQL\14\bin" + $env:PATH = "$PG_BIN;$env:PATH" + Write-Host "PATH=$env:PATH" + + pg_ctl start + Start-Sleep -Seconds 15 + + cd database + bash setup.sh + + - name: Setup redis (on Windows) + if: runner.os == 'Windows' + run: | + $URL = "https://github.com/redis-windows/redis-windows/releases/download/7.0.14/Redis-7.0.14-Windows-x64-with-Service.tar.gz" + $outputFolder = "C:\redis" + + if (-not (Test-Path $outputFolder)) { + New-Item -ItemType Directory -Path $outputFolder | Out-Null + } + + Invoke-WebRequest -Uri $URL -OutFile "redis.tar.gz" + tar -xf "redis.tar.gz" -C $outputFolder + $innerFolder = Get-ChildItem -Path $outputFolder | Select-Object -First 1 + Move-Item -Path "$outputFolder\$($innerFolder.Name)\*" -Destination $outputFolder -Force + Remove-Item -Path "$outputFolder\$($innerFolder.Name)" -Force + + cd $outputFolder + sc.exe create Redis binpath=C:\redis\RedisService.exe start= auto + net start Redis + + - name: Start docker containers + if: runner.os == 'Linux' + run: | + docker-compose -f test-docker-compose.yml up -d pg-example redis-example + # sleep 15 # wait for database to be ready + # docker ps + + - name: Wait for PostgreSQL to become healthy + if: runner.os == 'Linux' + run: | + while [[ "$(docker inspect --format='{{.State.Health.Status}}' pg-example)" != "healthy" ]]; do + echo "Waiting for PostgreSQL container to become healthy..." + sleep 5 + done + timeout-minutes: 5 + + - name: Wait for Redis to become healthy + if: runner.os == 'Linux' + run: | + while [[ "$(docker inspect --format='{{.State.Health.Status}}' redis-example)" != "healthy" ]]; do + echo "Waiting for Redis container to become healthy..." + sleep 5 + done + timeout-minutes: 5 + - name: Use Node.js ${{ matrix.node }} uses: actions/setup-node@v3 with: node-version: ${{ matrix.node }} + - uses: actions/cache@v3 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} restore-keys: | ${{ runner.os }}-node- + - run: npm ci + - run: npm run dotest env: MODE: test + + - name: Stop containers + if: runner.os == 'Linux' + run: docker-compose down diff --git a/.prettierignore b/.prettierignore index 62176518..9e8f3bcb 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ application/tasks/ package-lock.json package.json +test-docker-compose.yml diff --git a/Dockerfile b/Dockerfile index bc6db0e0..a9563012 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16-alpine +FROM node:20-alpine WORKDIR /usr/server COPY . . RUN npm ci --only=production diff --git a/application/db/redis/start.js b/application/db/redis/start.js index 30433c28..792a11c9 100644 --- a/application/db/redis/start.js +++ b/application/db/redis/start.js @@ -10,5 +10,5 @@ async () => { } client.quit(); }); - //await client.connect(); + await client.connect(); }; diff --git a/docker-compose.yml b/docker-compose.yml index 7624c971..1ea498c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: container_name: api-example environment: - DB_HOST=pg-example + - REDIS_HOST=redis-example volumes: - ./application:/usr/server/application depends_on: @@ -19,7 +20,7 @@ services: restart: always pg-example: - image: postgres:14.1-alpine3.15 + image: postgres:16-alpine container_name: pg-example environment: - POSTGRES_USER=marcus @@ -33,6 +34,15 @@ services: - 127.0.0.1:5432:5432 restart: always + redis-example: + image: redis:alpine + container_name: redis-example + volumes: + - ./data/redis-example:/data + ports: + - 127.0.0.1:6379:6379 + restart: always + networks: default: name: api-example-network diff --git a/test-docker-compose.yml b/test-docker-compose.yml new file mode 100644 index 00000000..8d1a5f3c --- /dev/null +++ b/test-docker-compose.yml @@ -0,0 +1,37 @@ +version: '3' + +services: + pg-example: + image: postgres:16-alpine + container_name: pg-example + environment: + - POSTGRES_USER=marcus + - POSTGRES_PASSWORD=marcus + - POSTGRES_DB=application + volumes: + - ./database/structure.sql:/docker-entrypoint-initdb.d/1.sql + - ./database/data.sql:/docker-entrypoint-initdb.d/2.sql + ports: + - 127.0.0.1:5432:5432 + restart: always + healthcheck: + test: ["CMD", "pg_isready", "-q", "-h", "localhost", "-p", "5432", "-U", "marcus"] + interval: 10s + timeout: 5s + retries: 3 + + redis-example: + image: redis:alpine + container_name: redis-example + ports: + - 127.0.0.1:6379:6379 + restart: always + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 3 + +networks: + default: + name: api-example-network From a00a1ab2633e8618af759eada256a7ac06b6368d Mon Sep 17 00:00:00 2001 From: Heorhii Date: Sun, 31 Dec 2023 03:05:52 +0100 Subject: [PATCH 3/6] Removed commented strings --- .github/workflows/test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0de20eaa..9b9dbd8f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,6 @@ jobs: - name: Install PostgreSQL and Redis (on MacOS) if: runner.os == 'macOS' run: | - # brew install postgresql brew install redis brew services start postgresql brew services start redis @@ -30,7 +29,6 @@ jobs: - name: Check services status (on MacOS) if: runner.os == 'macOS' run: | - # until brew services list | grep -E 'postgresql[[:space:]]+started' && brew services list | grep -E 'redis[[:space:]]+started'; do sleep 5; done until pg_isready -h localhost -p 5432 && redis-cli ping; do sleep 5; done - name: Set up PostgreSQL (on MacOS) @@ -79,8 +77,6 @@ jobs: if: runner.os == 'Linux' run: | docker-compose -f test-docker-compose.yml up -d pg-example redis-example - # sleep 15 # wait for database to be ready - # docker ps - name: Wait for PostgreSQL to become healthy if: runner.os == 'Linux' From 8089f92ff78ceb120ba632fe2a03b7fc50674a1d Mon Sep 17 00:00:00 2001 From: Heorhii Date: Fri, 26 Jan 2024 08:19:28 +0100 Subject: [PATCH 4/6] Refactored --- .github/workflows/test.yml | 39 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b9dbd8f..aeffc0fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,43 +19,41 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install PostgreSQL and Redis (on MacOS) + - name: Setup (on MacOS) if: runner.os == 'macOS' run: | + # install and up brew install redis brew services start postgresql brew services start redis - - name: Check services status (on MacOS) - if: runner.os == 'macOS' - run: | + # wait for ready until pg_isready -h localhost -p 5432 && redis-cli ping; do sleep 5; done - - name: Set up PostgreSQL (on MacOS) - if: runner.os == 'macOS' - run: | + # setup db createuser -s postgres cd database bash setup.sh - - name: Setup pg (on Windows) + - name: Setup (on Windows) if: runner.os == 'Windows' run: | + # postgres echo "unix_socket_directories = ''" >> "$PGDATA/postgresql.conf" echo "port = 5432" >> "$PGDATA/postgresql.conf" $PG_BIN = "C:\Program Files\PostgreSQL\14\bin" $env:PATH = "$PG_BIN;$env:PATH" Write-Host "PATH=$env:PATH" + # start and wait pg_ctl start Start-Sleep -Seconds 15 + # setup db cd database bash setup.sh - - name: Setup redis (on Windows) - if: runner.os == 'Windows' - run: | + # redis $URL = "https://github.com/redis-windows/redis-windows/releases/download/7.0.14/Redis-7.0.14-Windows-x64-with-Service.tar.gz" $outputFolder = "C:\redis" @@ -69,32 +67,23 @@ jobs: Move-Item -Path "$outputFolder\$($innerFolder.Name)\*" -Destination $outputFolder -Force Remove-Item -Path "$outputFolder\$($innerFolder.Name)" -Force + # start redis cd $outputFolder sc.exe create Redis binpath=C:\redis\RedisService.exe start= auto net start Redis - - name: Start docker containers + - name: Setup (on Linux) if: runner.os == 'Linux' run: | docker-compose -f test-docker-compose.yml up -d pg-example redis-example - - - name: Wait for PostgreSQL to become healthy - if: runner.os == 'Linux' - run: | while [[ "$(docker inspect --format='{{.State.Health.Status}}' pg-example)" != "healthy" ]]; do - echo "Waiting for PostgreSQL container to become healthy..." - sleep 5 + echo "Waiting for PostgreSQL container to become healthy..." + sleep 5 done - timeout-minutes: 5 - - - name: Wait for Redis to become healthy - if: runner.os == 'Linux' - run: | while [[ "$(docker inspect --format='{{.State.Health.Status}}' redis-example)" != "healthy" ]]; do echo "Waiting for Redis container to become healthy..." sleep 5 done - timeout-minutes: 5 - name: Use Node.js ${{ matrix.node }} uses: actions/setup-node@v3 @@ -114,6 +103,6 @@ jobs: env: MODE: test - - name: Stop containers + - name: Stop containers for Linux if: runner.os == 'Linux' run: docker-compose down From c20ba52a6c804475a3b262fff0896c75c0f9a4d3 Mon Sep 17 00:00:00 2001 From: Heorhii Date: Fri, 26 Jan 2024 09:12:54 +0100 Subject: [PATCH 5/6] Fixed deprecated Node.js 16 actions --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aeffc0fe..3e66294b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: - macos-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup (on MacOS) if: runner.os == 'macOS' @@ -86,11 +86,11 @@ jobs: done - name: Use Node.js ${{ matrix.node }} - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.npm key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} From 1eacd004fa2b264a7d9735c93f3ebc2087c83558 Mon Sep 17 00:00:00 2001 From: Heorhii Date: Fri, 26 Jan 2024 10:26:01 +0100 Subject: [PATCH 6/6] Added waiting instead of sleep --- .github/workflows/test.yml | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3e66294b..24cbf218 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,11 +43,21 @@ jobs: echo "port = 5432" >> "$PGDATA/postgresql.conf" $PG_BIN = "C:\Program Files\PostgreSQL\14\bin" $env:PATH = "$PG_BIN;$env:PATH" - Write-Host "PATH=$env:PATH" # start and wait pg_ctl start - Start-Sleep -Seconds 15 + $timeout = 120 + $elapsedTime = 0 + Write-Host "Waiting for PostgreSQL to start" + while ($elapsedTime -lt $timeout) { + $pgIsReadyResult = & $PG_BIN\pg_isready.exe -h localhost -p 5432 -t 1 + if ($pgIsReadyResult -eq "localhost:5432 - accepting connections") { + Write-Host "PostgreSQL is ready" + break + } + Start-Sleep -Seconds 5 + $elapsedTime += 5 + } # setup db cd database @@ -72,6 +82,25 @@ jobs: sc.exe create Redis binpath=C:\redis\RedisService.exe start= auto net start Redis + # wait for redis + Write-Host "Waiting for Redis to start" + while ($elapsedTime -lt $timeout) { + $redisPingResult = & C:\redis\redis-cli.exe ping + if ($redisPingResult -eq "PONG") { + Write-Host "Redis is ready" + break + } + Start-Sleep -Seconds 5 + $elapsedTime += 5 + } + + if ($elapsedTime -ge $timeout) { + Write-Host "Timeout waiting for services to start." + exit 1 + } + + Write-Host "Services are ready" + - name: Setup (on Linux) if: runner.os == 'Linux' run: |