diff --git a/.github/wait-for-it.sh b/.github/wait-for-it.sh new file mode 100755 index 00000000..d990e0d3 --- /dev/null +++ b/.github/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..a5885708 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,35 @@ +name: "CodeQL" + +on: + push: + branches: [ next, latest ] + pull_request: + branches: [ next ] + schedule: + - cron: '33 11 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 00000000..575400e3 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,19 @@ +name: Validation + +on: + release: + types: [prereleased] + + +jobs: + deploy: + name: "Deploy" + runs-on: ubuntu-latest + env: + REF: ${{ github.ref }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Dummy + run: echo "${REF:11}" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..457b39ab --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,44 @@ +name: Release + +on: + workflow_call: + inputs: + AWS_REGION: + required: true + AWS_EKS_CLUSTER_NAME: + required: true + RELEASE_NAME: + required: true + RELEASE_NAMESPACE: + required: true + VERSION: + required: true + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + +jobs: + deploy-to-eks: + name: Deploy to EKS + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.AWS_REGION }} + - name: Configure Kubectl + run: aws eks update-kubeconfig --name ${{ inputs.AWS_EKS_CLUSTER_NAME }} + - name: Install Helm + run: curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + - name: Deploy to cluster + run: | + helm upgrade ${{ inputs.RELEASE_NAME }} ./charts/app \ + --namespace ${{ inputs.RELEASE_NAMESPACE }} \ + --reuse-values \ + --set app.image.tag=${{ inputs.VERSION }} \ + --set app.global.publicPath=https://CLOUDID.cloudfront.net/${{ inputs.VERSION }} diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 00000000..5d9b4448 --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,17 @@ +name: Pull Request validation + +on: [pull_request] + +jobs: + # run the complete validation flow + validation: + uses: ./.github/workflows/validation.yml + secrets: + CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + +# for validation we only need to the validate the latest commit +# no need to continue with previous commits +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true diff --git a/.github/workflows/validate-and-release.yml b/.github/workflows/validate-and-release.yml new file mode 100644 index 00000000..fff7273d --- /dev/null +++ b/.github/workflows/validate-and-release.yml @@ -0,0 +1,47 @@ +name: Pull Request validation + +on: + push: + branches: + # latest channel + - stable + # next channel + - next + # specific versions + - '*.*.x' + +# for releasing we prefer to play safe and proceed with only one run at the time +concurrency: + group: ${{ github.workflow }} + +jobs: + # run the complete validation flow + validation: + uses: ./.github/workflows/validation.yml + secrets: + CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + + release: + name: Release + runs-on: ubuntu-latest + needs: validation + steps: + - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: 16.x + cache: 'yarn' +# - name: Configure AWS credentials +# uses: aws-actions/configure-aws-credentials@v1 +# with: +# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} +# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# aws-region: REGION +# - name: Fetch tokens for AWS ECR +# run: aws ecr get-login-password --region REGION | docker login --username AWS --password-stdin ACCOUNTID.dkr.ecr.REGION.amazonaws.com + - name: Semantic Release + run: yarn semantic-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml new file mode 100644 index 00000000..73c1883c --- /dev/null +++ b/.github/workflows/validation.yml @@ -0,0 +1,233 @@ +name: Validation + +on: + workflow_call: + secrets: + CYPRESS_PROJECT_ID: + required: true + CYPRESS_RECORD_KEY: + required: true + + +jobs: + install-dependencies: + name: Install dependencies + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: 16.x + cache: 'yarn' + - name: Install dependencies + run: yarn install --frozen-lockfile + + build: + name: Build + runs-on: ubuntu-latest + needs: install-dependencies + steps: + - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: 16.x + cache: 'yarn' + - name: Install dependencies + run: yarn install --frozen-lockfile + - name: Build + run: yarn build + env: + NODE_ENV: production + - uses: actions/upload-artifact@v2 + name: Upload build artifact + with: + name: build + path: build/ + retention-days: 7 + - uses: actions/upload-artifact@v2 + name: Upload report artifact + with: + name: report + path: report.html + retention-days: 7 + + unit-testing: + name: Unit testing + runs-on: ubuntu-latest + needs: install-dependencies + services: + mailhog: + image: mailhog/mailhog:v1.0.0 + ports: + - 1025:1025 + mongo: + image: circleci/mongo:5.0.3 + env: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: password + ports: + - 27017:27017 + minio: + image: bitnami/minio:2021.12.29-debian-10-r39 + env: + MINIO_ROOT_USER: 'AKIAIOSFODNN7EXAMPLE' + MINIO_ROOT_PASSWORD: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' + ports: + - 9000:9000 + redis: + image: circleci/redis:6.2.4-alpine + ports: + - 6379:6379 + html2pdf: + image: appvantage/html2pdf-service:1.3.0 + ports: + - 4000:3000 + steps: + - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: 16.x + cache: 'yarn' + - name: Install dependencies + run: yarn install --frozen-lockfile + - name: Wait for db + run: ./.github/wait-for-it.sh 127.0.0.1:27017 -t 60 + - name: Unit testing + run: yarn test:coverage --reporters=default --reporters=jest-junit + env: + # app config + APP_DB_URI: mongodb://root:password@127.0.0.1:27017 + APP_SESSION_SECRET: localSecret + APP_FLOW_SECRET: flowSecret + APP_SMTP_PORT: 1025 + APP_STORAGE_ENDPOINT: localhost + APP_STORAGE_PORT: 9000 + APP_STORAGE_SSL: false + APP_STORAGE_ACCESS_KEY: 'AKIAIOSFODNN7EXAMPLE' + APP_STORAGE_SECRET_KEY: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' + APP_HTML2PDF_ENDPOINT: 'http://127.0.0.1:4000/' + APP_DEBUG: true + # jest config + JEST_JUNIT_OUTPUT_DIR: ./junit/ + - name: Upload coverage artifact + uses: actions/upload-artifact@v2 + with: + name: coverage + path: coverage/ + retention-days: 7 + + functional-testing: + name: Functional testing + runs-on: ubuntu-latest + needs: build + env: + APP_DB_URI: mongodb://root:password@127.0.0.1:27017 + APP_SESSION_SECRET: localSecret + APP_FLOW_SECRET: flowSecret + APP_SMTP_PORT: 1025 + APP_STORAGE_ENDPOINT: localhost + APP_STORAGE_PORT: 9000 + APP_STORAGE_SSL: false + APP_STORAGE_ACCESS_KEY: 'AKIAIOSFODNN7EXAMPLE' + APP_STORAGE_SECRET_KEY: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' + APP_HTML2PDF_ENDPOINT: 'http://127.0.0.1:4000/' + APP_DEBUG: true + services: + mailhog: + image: mailhog/mailhog:v1.0.0 + ports: + - 1025:1025 + mongo: + image: circleci/mongo:5.0.3 + env: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: password + ports: + - 27017:27017 + minio: + image: bitnami/minio:2021.12.29-debian-10-r39 + env: + MINIO_ROOT_USER: 'AKIAIOSFODNN7EXAMPLE' + MINIO_ROOT_PASSWORD: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' + ports: + - 9000:9000 + redis: + image: circleci/redis:6.2.4-alpine + ports: + - 6379:6379 + html2pdf: + image: appvantage/html2pdf-service:1.3.0 + ports: + - 4000:3000 + steps: + - uses: actions/checkout@v2 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: 16.x + cache: 'yarn' + - name: Install dependencies + run: yarn install --frozen-lockfile + - name: Wait for db + run: ./.github/wait-for-it.sh 127.0.0.1:27017 -t 60 + - name: Download build artifact + uses: actions/download-artifact@v2 + with: + name: build + path: build/ + - name: Start server + run: node server.js serve &>/dev/null & + working-directory: ./build + - name: Start worker + run: node server.js worker &>/dev/null & + working-directory: ./build + - name: Wait for test server + run: ./.github/wait-for-it.sh 127.0.0.1:3000 -t 60 + - name: Cypress run + uses: cypress-io/github-action@v2 + with: + record: true + env: + DOTENV_DISABLE: true + CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + CYPRESS_BASE_URL: "http://localhost:3000" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload cypress videos + uses: actions/upload-artifact@v2 + with: + name: cypress-videos + path: cypress/videos/ + retention-days: 7 + - name: Stop test server & worker + run: pkill -SIGQUIT node + + linting-and-typing: + name: "Linting & Typing" + runs-on: ubuntu-latest + needs: install-dependencies + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Use Node.js + uses: actions/setup-node@v2 + with: + node-version: 16.x + cache: 'yarn' + - name: Install dependencies + run: yarn install --frozen-lockfile + - name: Lint source code + run: yarn lint --format junit -o ./junit/js-lint-results.xml + - name: Type checking + run: yarn tsc + - name: Type checking on cypress + run: yarn tsc + working-directory: ./cypress + - name: Lint commit messages + run: node ./devtools/commands/lint-commits.js + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/devtools/commands/lint-commits.js b/devtools/commands/lint-commits.js index 65198134..4d57c76f 100644 --- a/devtools/commands/lint-commits.js +++ b/devtools/commands/lint-commits.js @@ -26,39 +26,57 @@ const matchGithub = (url, prop) => { }; const getRangeFromPr = async () => { - const { owner, repo, data: pull } = matchGithub(process.env.CIRCLE_PULL_REQUEST, 'pull'); - const github = new Octokit({ auth: process.env.GH_TOKEN }); + const ghToken = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; - console.info('📡 Looking up PR #%s...', pull); + if (!ghToken) { + throw new Error('GitHub token is missing'); + } + + if (process.env.CIRCLE_PULL_REQUEST) { + // use CircleCI setup + const { owner, repo, data: pull } = matchGithub(process.env.CIRCLE_PULL_REQUEST, 'pull'); + const github = new Octokit({ auth: ghToken }); - const { - data: { base, head }, - } = await github.pulls.get({ owner, repo, pull_number: +pull }); + console.info('📡 Looking up PR #%s...', pull); - await checkCommit(base.sha, head.sha); - console.info('🔀 Linting PR #%s', pull); + const { + data: { base, head }, + } = await github.pulls.get({ owner, repo, pull_number: +pull }); - return [base.sha, head.sha]; + await checkCommit(base.sha, head.sha); + console.info('🔀 Linting PR #%s', pull); + + return [base.sha, head.sha]; + } + + throw new Error('Environment not supported to get a range'); }; const getRangeFromCompare = async () => { - const [from, to] = matchGithub(process.env.CIRCLE_COMPARE_URL, 'compare').data.split('...'); + if (process.env.CIRCLE_COMPARE_URL) { + // use CircleCI setup + const [from, to] = matchGithub(process.env.CIRCLE_COMPARE_URL, 'compare').data.split('...'); + + await checkCommit(from, to); + console.info('🎏 Linting using comparison URL %s...%s', from, to); - await checkCommit(from, to); - console.info('🎏 Linting using comparison URL %s...%s', from, to); + return [from, to]; + } - return [from, to]; + throw new Error('Environment not supported to identity the comparison'); }; const getRangeFromSha = async () => { - const sha = process.env.CIRCLE_SHA1; + // for CircleCI get the commit ID from `CIRCLE_SHA1` + // for GH Actions get the commit ID from `GITHUB_SHA` + const sha = process.env.CIRCLE_SHA1 || process.env.GITHUB_SHA; if (!sha) { - throw new Error('Cannot find CIRCLE_SHA1 environment variable'); + throw new Error('Cannot find the git commit ID in environment variable'); } await checkCommit(sha); - console.info('⚙️ Linting using CIRCLE_SHA1 (%s)', sha); + console.info('⚙️ Linting using commit ID (%s)', sha); return [`${sha}^1`, sha]; }; diff --git a/devtools/env.js b/devtools/env.js index c720dd2b..0c645e94 100644 --- a/devtools/env.js +++ b/devtools/env.js @@ -55,15 +55,17 @@ const loadEnvConfig = (dir, dev = false, verbose = true) => { const isTest = process.env.NODE_ENV === 'test'; const mode = getMode(isTest, dev); - const dotenvFiles = [ - `.env.${mode}.local`, - // Don't include `.env.local` for `test` environment - // since normally you expect tests to produce the same - // results for everyone - mode !== 'test' && `.env.local`, - `.env.${mode}`, - '.env', - ].filter(Boolean); + const dotenvFiles = process.env.DOTENV_DISABLE + ? [] + : [ + `.env.${mode}.local`, + // Don't include `.env.local` for `test` environment + // since normally you expect tests to produce the same + // results for everyone + mode !== 'test' && `.env.local`, + `.env.${mode}`, + '.env', + ].filter(Boolean); for (const envFile of dotenvFiles) { // only load .env if the user provided has an env config file