From f48c4684ba2fa8d477329e21db8216b559cef374 Mon Sep 17 00:00:00 2001 From: Daniel Genis Date: Thu, 19 Oct 2023 12:16:59 +0200 Subject: [PATCH 1/4] feat: add master merge support --- .github.jsonnet | 45 +-- .github/jsonnet/GIT_VERSION | 2 +- .github/jsonnet/base.jsonnet | 88 ++++-- .github/jsonnet/clusters.jsonnet | 11 +- .github/jsonnet/complete-workflows.jsonnet | 31 ++ .github/jsonnet/databases.jsonnet | 144 ++++++---- .github/jsonnet/deployment.jsonnet | 89 ++++++ .github/jsonnet/docker.jsonnet | 32 ++- .github/jsonnet/helm.jsonnet | 288 +++++++++++-------- .github/jsonnet/images.jsonnet | 20 +- .github/jsonnet/index.jsonnet | 20 +- .github/jsonnet/misc.jsonnet | 252 +++++++++++----- .github/jsonnet/mongo.jsonnet | 65 +++++ .github/jsonnet/newrelic.jsonnet | 31 +- .github/jsonnet/notifications.jsonnet | 27 ++ .github/jsonnet/pubsub.jsonnet | 31 +- .github/jsonnet/pull-upstream-and-rebuild.sh | 2 +- .github/jsonnet/pulumi.jsonnet | 154 ++++++++++ .github/jsonnet/ruby.jsonnet | 257 +++++++++-------- .github/jsonnet/services.jsonnet | 120 ++++---- .github/jsonnet/yarn.jsonnet | 249 +++++++++++----- .github/workflows/misc.yml | 28 ++ .github/workflows/pr.yml | 139 +++++++++ .github/workflows/publish-prod.yml | 74 +++++ .github/workflows/publish-tag.yml | 29 -- .github/workflows/test-pr.yml | 31 -- 26 files changed, 1578 insertions(+), 681 deletions(-) create mode 100644 .github/jsonnet/complete-workflows.jsonnet create mode 100644 .github/jsonnet/deployment.jsonnet create mode 100644 .github/jsonnet/mongo.jsonnet create mode 100644 .github/jsonnet/notifications.jsonnet create mode 100644 .github/jsonnet/pulumi.jsonnet create mode 100644 .github/workflows/misc.yml create mode 100644 .github/workflows/pr.yml create mode 100644 .github/workflows/publish-prod.yml delete mode 100644 .github/workflows/publish-tag.yml delete mode 100644 .github/workflows/test-pr.yml diff --git a/.github.jsonnet b/.github.jsonnet index fce8ecc..66f56fb 100644 --- a/.github.jsonnet +++ b/.github.jsonnet @@ -1,38 +1,17 @@ local util = import './.github/jsonnet/index.jsonnet'; local image = 'eu.gcr.io/unicorn-985/docker-images_node14-with-libnss:deploy-5893c6fca68ea35a0a51e855d5a3cb7082ef39fa'; -util.pipeline( +local testJob = util.ghJob( 'test-pr', - [ - util.ghJob( - 'test-pr', - image=image, - useCredentials=true, - steps=[ - util.checkoutAndYarn(ref='${{ github.event.pull_request.head.sha }}', fullClone=false), - util.action('setup chrome', 'browser-actions/setup-chrome@latest'), - util.step('test', './node_modules/.bin/ember test'), - util.yarnPublish(isPr=true), - ], - runsOn=['ubuntu-latest'], // it's public fork. don't use private runners for public fork - ), + image=image, + useCredentials=true, + steps=[ + util.checkoutAndYarn(ref='${{ github.event.pull_request.head.sha }}', fullClone=false), + util.action('setup chrome', 'browser-actions/setup-chrome@latest'), + util.step('test', './node_modules/.bin/ember test'), + util.yarnPublish(isPr=true), ], -) + -util.pipeline( - 'publish-tag', - [ - util.ghJob( - 'publish', - image=image, - useCredentials=true, - steps=[ - util.checkoutAndYarn(ref='${{ github.sha }}', fullClone=false), - util.yarnPublish(isPr=false), - ], - runsOn=['ubuntu-latest'], // it's public fork. don't use private runners for public fork - ), - ], - event={ - push: { tags: ['*'] }, - }, -) + runsOn=['ubuntu-latest'], // it's public fork. don't use private runners for public fork +); + +util.workflowJavascriptPackage(testJob=testJob) diff --git a/.github/jsonnet/GIT_VERSION b/.github/jsonnet/GIT_VERSION index a61bc12..f11077c 100644 --- a/.github/jsonnet/GIT_VERSION +++ b/.github/jsonnet/GIT_VERSION @@ -1 +1 @@ -497456f307c7f9ef1d9ac1660b951b1cf8a6e024 +9f0090f332fe3306ec145c60d2e129628aee2f9c diff --git a/.github/jsonnet/base.jsonnet b/.github/jsonnet/base.jsonnet index 38dceff..5f7ef5c 100644 --- a/.github/jsonnet/base.jsonnet +++ b/.github/jsonnet/base.jsonnet @@ -1,16 +1,17 @@ { - pipeline(name, jobs, event = 'pull_request', permissions = null):: { + pipeline(name, jobs, event=['pull_request'], permissions=null, concurrency=null):: { [name + '.yml']: std.manifestYamlDoc( { name: name, on: event, jobs: std.foldl(function(x, y) x + y, jobs, {}), - } + (if permissions == null then {} else {permissions: permissions}), - ), + } + (if permissions == null then {} else { permissions: permissions }) + (if concurrency == null then {} else { concurrency: concurrency }), + ), }, - ghJob(name, + ghJob( + name, timeoutMinutes=30, runsOn=null, image=$.default_job_image, @@ -20,40 +21,69 @@ outputs=null, useCredentials=true, services=null, + permissions=null, + concurrency=null, + continueOnError=null, ):: - { - [name]: { - 'timeout-minutes': timeoutMinutes, - 'runs-on': (if runsOn == null then ['self-hosted', 'runner-2'] else runsOn), - container: { - image: image, - } + (if useCredentials then { credentials: { username: '_json_key', password: $.secret('docker_gcr_io') }} else {}) - , - steps: std.flattenArrays(steps), - } + - (if ifClause != null then {'if': ifClause} else {}) + - (if needs != null then {'needs': needs} else {}) + - (if outputs != null then {'outputs': outputs} else {}) + - (if services != null then {'services': services} else {}) - }, + { + [name]: { + 'timeout-minutes': timeoutMinutes, + 'runs-on': (if runsOn == null then ['self-hosted', 'runner-2'] else runsOn), + } + + ( + if image == null then {} else + { + container: { + image: image, + } + (if useCredentials then { credentials: { username: '_json_key', password: $.secret('docker_gcr_io') } } else {}), + } + ) + + { + steps: std.flattenArrays(steps), + } + + (if ifClause != null then { 'if': ifClause } else {}) + + (if needs != null then { needs: needs } else {}) + + (if outputs != null then { outputs: outputs } else {}) + + (if services != null then { services: services } else {}) + + (if permissions == null then {} else { permissions: permissions }) + + (if concurrency == null then {} else { concurrency: concurrency }) + + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }), + }, + + ghExternalJob( + name, + uses, + with=null, + ):: + { + [name]: { + uses: uses, + } + (if with != null then { + with: with, + } else {}), + }, - step(name, run, env=null, workingDirectory=null, ifClause=null, id=null):: - [{ - name: name, - run: run, - } + (if workingDirectory != null then { 'working-directory': workingDirectory } else { }) + step(name, run, env=null, workingDirectory=null, ifClause=null, id=null, continueOnError=null):: + [ + { + name: name, + run: run, + } + (if workingDirectory != null then { 'working-directory': workingDirectory } else {}) + (if env != null then { env: env } else {}) + (if ifClause != null then { 'if': ifClause } else {}) + (if id != null then { id: id } else {}) + + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }), ], - action(name, uses, env=null, with=null, id=null, ifClause=null):: - [{ - name: name, - uses: uses, - } + (if env != null then { env: env } else {}) + action(name, uses, env=null, with=null, id=null, ifClause=null, continueOnError=null):: + [ + { + name: name, + uses: uses, + } + (if env != null then { env: env } else {}) + (if with != null && with != {} then { with: with } else {}) + (if id != null then { id: id } else {}) + (if ifClause != null then { 'if': ifClause } else {}) + + (if continueOnError == null then {} else { 'continue-on-error': continueOnError }), ], } diff --git a/.github/jsonnet/clusters.jsonnet b/.github/jsonnet/clusters.jsonnet index fb51356..3ebe0e7 100644 --- a/.github/jsonnet/clusters.jsonnet +++ b/.github/jsonnet/clusters.jsonnet @@ -12,6 +12,13 @@ name: 'prod-europe-west4', zone: 'europe-west4', secret: '${{ secrets.GCE_JSON }}', - } - } + }, + + 'gynzy-intern': { + project: 'gynzy-intern', + name: 'gynzy-intern', + zone: 'europe-west4', + secret: '${{ secrets.CONTINUOUS_DEPLOYMENT_GCE_JSON }}', + }, + }, } diff --git a/.github/jsonnet/complete-workflows.jsonnet b/.github/jsonnet/complete-workflows.jsonnet new file mode 100644 index 0000000..b4735fe --- /dev/null +++ b/.github/jsonnet/complete-workflows.jsonnet @@ -0,0 +1,31 @@ +{ + /* + @param {string[]} repositories - The repositories to publish to + @param {boolean} isPublicFork - Whether the repository is a public fork + @param {boolean} checkVersionBump - Whether to assert if the version was bumped (recommended) + @param {ghJob} testJob - a job to be ran during PR to assert tests. can be an array of jobs + */ + workflowJavascriptPackage(repositories=['gynzy'], isPublicFork=true, checkVersionBump=true, testJob=null):: + local runsOn = (if isPublicFork then 'ubuntu-latest' else null); + + $.pipeline( + 'misc', + [$.verifyJsonnet(fetch_upstream=false, runsOn=runsOn)], + ) + + $.pipeline( + 'publish-prod', + [ + $.yarnPublishJob(repositories, runsOn=runsOn), + ], + event={ push: { branches: ['${{ github.event.pull_request.base.ref }}'] } }, + ) + + $.pipeline( + 'pr', + [ + $.yarnPublishPreviewJob(repositories=repositories, runsOn=runsOn, checkVersionBump=checkVersionBump), + ] + + (if testJob != null then + [testJob] + else []) + ), +} diff --git a/.github/jsonnet/databases.jsonnet b/.github/jsonnet/databases.jsonnet index 751f2dc..d8af183 100644 --- a/.github/jsonnet/databases.jsonnet +++ b/.github/jsonnet/databases.jsonnet @@ -1,76 +1,106 @@ { database_servers: { - test: { - server: 'test-ams', - region: 'europe-west4', - project: 'unicorn-985', - }, - 'test-ams-8': { - server: 'test-ams-8', - region: 'europe-west4', - project: 'unicorn-985', - }, - 'eu-w4-unicorn-production': { - server: 'eu-w4-unicorn-production', - region: 'europe-west4', - project: 'unicorn-985', - }, - 'eu-w4-responses-production' : { - server: 'eu-w4-responses-production', - region: 'europe-west4', - project: 'unicorn-985', - }, - 'eu-w4-metrics-production' : { - server: 'eu-w4-metrics-production', - region: 'europe-west4', - project: 'unicorn-985', - }, - 'gynzy-test' : { - server: 'gynzy-test', - region: 'europe-west4', - project: 'gynzy-1090', - }, - 'gynzy-production' : { - server: 'gynzy-production', - region: 'europe-west4', - project: 'gynzy-1090', - }, - 'accounts-production': { - server: 'eu-w4-accounts-production', - region: 'europe-west4', - project: 'unicorn-985', - }, - 'eu-w4-licenses-v8' : { - server: 'eu-w4-licenses-v8', - region: 'europe-west4', - project: 'unicorn-985', - }, - 'eu-w4-curriculum-v8': { - server: 'eu-w4-curriculum-v8', - region: 'europe-west4', - project: 'unicorn-985', - } + test: { + type: 'mysql', + server: 'test-ams', + region: 'europe-west4', + project: 'unicorn-985', + lifecycle: 'test', + }, + 'test-ams-8': { + type: 'mysql', + server: 'test-ams-8', + region: 'europe-west4', + project: 'unicorn-985', + lifecycle: 'test', + }, + 'eu-w4-unicorn-production': { + type: 'mysql', + server: 'eu-w4-unicorn-production', + region: 'europe-west4', + project: 'unicorn-985', + }, + 'eu-w4-responses-production': { + type: 'mysql', + server: 'eu-w4-responses-production', + region: 'europe-west4', + project: 'unicorn-985', + }, + 'eu-w4-metrics-production': { + type: 'mysql', + server: 'eu-w4-metrics-production', + region: 'europe-west4', + project: 'unicorn-985', + }, + 'gynzy-test': { + type: 'mysql', + server: 'gynzy-test', + region: 'europe-west4', + project: 'gynzy-1090', + lifecycle: 'test', + }, + 'gynzy-production': { + type: 'mysql', + server: 'gynzy-production', + region: 'europe-west4', + project: 'gynzy-1090', + }, + 'accounts-production': { + type: 'mysql', + server: 'eu-w4-accounts-production', + region: 'europe-west4', + project: 'unicorn-985', + }, + 'eu-w4-licenses-v8': { + type: 'mysql', + server: 'eu-w4-licenses-v8', + region: 'europe-west4', + project: 'unicorn-985', + }, + 'eu-w4-curriculum-v8': { + type: 'mysql', + server: 'eu-w4-curriculum-v8', + region: 'europe-west4', + project: 'unicorn-985', + }, + 'eu-w4-enrollments-v8': { + type: 'mysql', + server: 'eu-w4-enrollments-v8', + region: 'europe-west4', + project: 'unicorn-985', + }, + 'eu-w4-board-v8': { + type: 'mysql', + server: 'eu-w4-board-v8', + region: 'europe-west4', + project: 'unicorn-985', + }, + }, copyDatabase(mysqlActionOptions):: - assert std.length(std.findSubstr('_pr_', mysqlActionOptions.database_name_target)) > 0; // target db gets deleted. must contain _pr_ + assert std.length(std.findSubstr('_pr_', mysqlActionOptions.database_name_target)) > 0; // target db gets deleted. must contain _pr_ // overwrite and set task to clone // delete database by setting it to null and calling prune afterwards - local pluginOptions = std.prune(mysqlActionOptions + { task: 'clone', database: null }); + local pluginOptions = std.prune(mysqlActionOptions { task: 'clone', database: null }); - $.action('copy-database', $.mysql_action_image, + $.action( + 'copy-database', + $.mysql_action_image, with=pluginOptions ), deleteDatabase(mysqlActionOptions):: - assert std.length(std.findSubstr('_pr_', mysqlActionOptions.database_name_target)) > 0; // this fn deletes the database. destination db must contain _pr_ + assert std.length(std.findSubstr('_pr_', mysqlActionOptions.database_name_target)) > 0; // this fn deletes the database. destination db must contain _pr_ // overwrite and set task to clone // delete database by setting it to null and calling prune afterwards - local pluginOptions = std.prune(mysqlActionOptions + { task: 'remove', database: null }); + local pluginOptions = std.prune(mysqlActionOptions { task: 'remove', database: null }); - $.action('delete-database', $.mysql_action_image, + $.action( + 'delete-database', + $.mysql_action_image, with=pluginOptions ), -} \ No newline at end of file +} diff --git a/.github/jsonnet/deployment.jsonnet b/.github/jsonnet/deployment.jsonnet new file mode 100644 index 0000000..057abde --- /dev/null +++ b/.github/jsonnet/deployment.jsonnet @@ -0,0 +1,89 @@ +{ + _assertMergeShaIsLatestCommit(branch, sha='${{ github.sha }}', repository='${{ github.repository }}'):: + [ + $.step('install jq curl', 'apk add --no-cache jq curl'), + $.step( + 'assert merge sha is latest commit', + ||| + HEAD_SHA=$(curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/${GITHUB_REPOSITORY}/branches/${TARGET_BRANCH} | jq -r .commit.sha); + if [ ${HEAD_SHA} == ${PR_MERGE_COMMIT_SHA} ]; then + echo "Merge sha is latest commit on branch ${TARGET_BRANCH}! HEAD_SHA: ${HEAD_SHA} PR_MERGE_COMMIT_SHA: ${PR_MERGE_COMMIT_SHA}"; + echo "CREATE_DEPLOY_EVENT=true" >> $GITHUB_OUTPUT + else + echo "Merge sha is not latest commit on branch ${TARGET_BRANCH}! HEAD_SHA: ${HEAD_SHA} PR_MERGE_COMMIT_SHA: ${PR_MERGE_COMMIT_SHA}"; + echo "CREATE_DEPLOY_EVENT=false" >> $GITHUB_OUTPUT + fi + |||, + env={ + PR_MERGE_COMMIT_SHA: sha, + GITHUB_REPOSITORY: repository, + TARGET_BRANCH: branch, + GITHUB_TOKEN: '${{ github.token }}', + }, + id='assert-merge-sha-is-latest-commit', + ), + ], + + masterMergeDeploymentEventHook(deployToTest=false, prodBranch=null, testBranch=null):: + $.pipeline( + 'create-merge-deployment', + [ + $.ghJob( + 'create-merge-deployment-prod', + useCredentials=false, + permissions={ deployments: 'write', contents: 'read' }, + ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", + steps=$._assertMergeShaIsLatestCommit(branch=(if prodBranch != null then prodBranch else '${{ github.event.pull_request.base.repo.default_branch }}')) + + [ + $.action( + 'publish-deploy-prod-event', + 'chrnorm/deployment-action@v2', + ifClause='${{ github.event.pull_request.base.ref == ' + (if prodBranch != null then "'" + prodBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", + with={ + token: $.secret('VIRKO_GITHUB_TOKEN'), + environment: 'production', + 'auto-merge': 'false', + ref: '${{ github.event.pull_request.head.sha }}', + description: 'Auto deploy production on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', + payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}" }', + } + ), + $.sendSlackMessage( + message='Deploy to prod of pr: ${{ github.event.number }} with title: ${{ github.event.pull_request.title }} branch: ${{ github.head_ref }} started!', + ifClause='${{ github.event.pull_request.base.ref == ' + (if prodBranch != null then "'" + prodBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", + ), + ], + ), + ] + + (if deployToTest == true then + [ + $.ghJob( + 'create-merge-deployment-test', + useCredentials=false, + permissions={ deployments: 'write', contents: 'read' }, + ifClause="${{ github.actor != 'gynzy-virko' && github.event.pull_request.merged == true}}", + steps=$._assertMergeShaIsLatestCommit(branch=(if testBranch != null then testBranch else '${{ github.event.pull_request.base.repo.default_branch }}')) + + [ + $.action( + 'publish-deploy-test-event', + 'chrnorm/deployment-action@v2', + ifClause='${{ github.event.pull_request.base.ref == ' + (if testBranch != null then "'" + testBranch + "'" else 'github.event.pull_request.base.repo.default_branch') + " && steps.assert-merge-sha-is-latest-commit.outputs.CREATE_DEPLOY_EVENT == 'true' }}", + with={ + token: $.secret('VIRKO_GITHUB_TOKEN'), + environment: 'test', + 'auto-merge': 'false', + ref: '${{ github.event.pull_request.head.sha }}', + description: 'Auto deploy test on PR merge. pr: ${{ github.event.number }} ref: ${{ github.event.pull_request.head.sha }}', + payload: '{ "pr" : ${{ github.event.number }}, "branch": "${{ github.head_ref }}" }', + } + ), + ], + ), + ] else []), + event={ + pull_request: { + types: ['closed'], + }, + }, + ), +} diff --git a/.github/jsonnet/docker.jsonnet b/.github/jsonnet/docker.jsonnet index 7404c45..cc03c3c 100644 --- a/.github/jsonnet/docker.jsonnet +++ b/.github/jsonnet/docker.jsonnet @@ -1,24 +1,26 @@ { - buildDocker(imageName, - imageTag = 'deploy-${{ github.event.pull_request.head.sha }}', - context = '.', - dockerfile = null, - env = {}, - build_args = null, - registry = 'eu.gcr.io', + buildDocker( + imageName, + imageTag='deploy-${{ github.event.pull_request.head.sha }}', + context='.', + dockerfile=null, + env={}, + build_args=null, + registry='eu.gcr.io', ):: $.action( 'build-docker', $.docker_action_image, with={ - context: context, - gcloud_service_key: $.secret('docker_gcr_io_base64'), - image_name: imageName, - image_tag: imageTag, - project_id: 'unicorn-985', - registry: registry, - } + if build_args != null then {build_args: build_args} else {} - + if dockerfile != null then {dockerfile: dockerfile} else {}, + context: context, + gcloud_service_key: $.secret('docker_gcr_io_base64'), + image_name: imageName, + image_tag: imageTag, + project_id: 'unicorn-985', + registry: registry, + } + + (if build_args != null then { build_args: build_args } else {}) + + (if dockerfile != null then { dockerfile: dockerfile } else {}), env=env, ), } diff --git a/.github/jsonnet/helm.jsonnet b/.github/jsonnet/helm.jsonnet index 1b347c0..8b7a871 100644 --- a/.github/jsonnet/helm.jsonnet +++ b/.github/jsonnet/helm.jsonnet @@ -1,195 +1,226 @@ { - deployHelm(cluster, release, values, chartPath, delete=false, useHelm3=false, title=null, ifClause=null):: - $.action((if title == null then if delete then 'delete-helm' else 'deploy-helm' else title), + deployHelm(cluster, release, values, chartPath, delete=false, useHelm3=true, title=null, ifClause=null, ttl=null, namespace='default'):: + $.action( + (if title == null then if delete then 'delete-helm' else 'deploy-helm' else title), $.helm_action_image, - with = { - 'clusterProject': cluster.project, - 'clusterLocation': cluster.zone, - 'clusterName': cluster.name, - 'clusterSaJson': cluster.secret, - release: release, - namespace: 'default', - chart: chartPath, - atomic: 'false', - token: '${{ github.token }}', - version: '${{ github.event.pull_request.head.sha }}', - values: if std.isString(values) then values else std.manifestJsonMinified(values), // Accepts a string and an object due to legacy reasons. - } + (if delete then {task: 'remove'} else {}) - + (if useHelm3 then {helm: 'helm3'} else { helm: 'helm' }), + with={ + clusterProject: cluster.project, + clusterLocation: cluster.zone, + clusterName: cluster.name, + clusterSaJson: cluster.secret, + release: release, + namespace: namespace, + chart: chartPath, + atomic: 'false', + token: '${{ github.token }}', + version: '${{ github.event.pull_request.head.sha }}', + values: if std.isString(values) then values else std.manifestJsonMinified(values), // Accepts a string and an object due to legacy reasons. + } + (if delete then { task: 'remove' } else {}) + + (if useHelm3 then { helm: 'helm3' } else { helm: 'helm' }) + + (if ttl != null then { ttl: ttl } else {}), ifClause=ifClause, ), - helmDeployProd(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-prod', - ifClause = null, + helmDeployProd( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-prod', + ifClause=null, + cluster=$.clusters.prod, + namespace='default', ):: $.deployHelm( - $.clusters.prod, + cluster, deploymentName, { - environment: "prod", - identifier: "prod", + environment: 'prod', + identifier: 'prod', image: { - tag: "deploy-${{ github.sha }}", + tag: 'deploy-${{ github.sha }}', }, } + options, helmPath, useHelm3=true, title='deploy-prod', ifClause=ifClause, + namespace=namespace, ), - helmDeployProdJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-prod', - image = $.default_job_image, - useCredentials = false, + helmDeployProdJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-prod', + image=$.default_job_image, + useCredentials=false, ):: - $.ghJob('deploy-prod', - ifClause='${{ github.event.deployment.environment == \'production\' }}', + $.ghJob( + 'deploy-prod', + ifClause="${{ github.event.deployment.environment == 'production' }}", image=image, useCredentials=useCredentials, steps=[ - $.checkout(), - $.helmDeployProd(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeployProd(serviceName, options, helmPath, deploymentName), ], ), - helmDeployTest(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-master', + helmDeployTest( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-master', + cluster=$.clusters.test, + namespace='default', ):: $.deployHelm( - $.clusters.test, + cluster, deploymentName, { - environment: "test", - identifier: "master", + environment: 'test', + identifier: 'master', image: { - tag: "deploy-${{ github.sha }}", + tag: 'deploy-${{ github.sha }}', }, } + options, helmPath, useHelm3=true, title='deploy-test', + namespace=namespace, ), - helmDeployTestJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-master', - image = $.default_job_image, - useCredentials = false, + helmDeployTestJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-master', + image=$.default_job_image, + useCredentials=false, ):: - $.ghJob('deploy-test', - ifClause='${{ github.event.deployment.environment == \'test\' }}', + $.ghJob( + 'deploy-test', + ifClause="${{ github.event.deployment.environment == 'test' }}", image=image, - useCredentials = useCredentials, + useCredentials=useCredentials, steps=[ - $.checkout(), - $.helmDeployTest(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeployTest(serviceName, options, helmPath, deploymentName), ], ), - helmDeployPR(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-pr-${{ github.event.number }}', + helmDeployPR( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-pr-${{ github.event.number }}', + cluster=$.clusters.test, + namespace='default', ):: $.deployHelm( - $.clusters.test, + cluster, deploymentName, { - environment: "pr", - identifier: "pr-${{ github.event.number }}", + environment: 'pr', + identifier: 'pr-${{ github.event.number }}', image: { - tag: "deploy-${{ github.event.pull_request.head.sha }}", + tag: 'deploy-${{ github.event.pull_request.head.sha }}', }, } + options, helmPath, useHelm3=true, title='deploy-pr', + ttl='7 days', + namespace=namespace, ), - helmDeployPRJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-pr-${{ github.event.number }}', - image = $.default_job_image, - useCredentials = false, + helmDeployPRJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-pr-${{ github.event.number }}', + image=$.default_job_image, + useCredentials=false, ):: - $.ghJob('deploy-pr', + $.ghJob( + 'deploy-pr', image=image, useCredentials=useCredentials, steps=[ - $.checkout(), - $.helmDeployPR(serviceName, options, helmPath, deploymentName), + $.checkout(), + $.helmDeployPR(serviceName, options, helmPath, deploymentName), ], ), - helmDeletePr(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-pr-${{ github.event.number }}', + helmDeletePr( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-pr-${{ github.event.number }}', + cluster=$.clusters.test, + namespace='default', ):: $.deployHelm( - $.clusters.test, + cluster, deploymentName, options, helmPath, useHelm3=true, delete=true, title='delete-pr', + namespace=namespace, ), - helmDeletePRJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-pr-${{ github.event.number }}', - mysqlDeleteOptions = { enabled: false }, + helmDeletePRJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-pr-${{ github.event.number }}', + mysqlDeleteOptions={ enabled: false }, ):: - $.ghJob('helm-delete-pr', + $.ghJob( + 'helm-delete-pr', image=$.default_job_image, useCredentials=false, steps=[ - $.checkout(), - $.helmDeletePr(serviceName, options, helmPath, deploymentName)] + - ( if mysqlDeleteOptions.enabled then [$.deleteDatabase(mysqlDeleteOptions)] else []), + $.checkout(), + $.helmDeletePr(serviceName, options, helmPath, deploymentName), + ] + + (if mysqlDeleteOptions.enabled then [$.deleteDatabase(mysqlDeleteOptions)] else []), services=(if mysqlDeleteOptions.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(mysqlDeleteOptions.database) } else null), ), - helmDeletePRPipeline(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-pr-${{ github.event.number }}', + helmDeletePRPipeline( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-pr-${{ github.event.number }}', ):: - $.pipeline('close-pr', + $.pipeline( + 'close-pr', [ $.helmDeletePRJob(serviceName, options, helmPath, deploymentName), ], - event = { + event={ pull_request: { - types: ['closed'] - } + types: ['closed'], + }, } ), - helmDeployCanary(serviceName, - options = {}, - helmPath = './helm/' + serviceName + '-canary', - deploymentName = serviceName + '-canary', - ifClause = null, + helmDeployCanary( + serviceName, + options={}, + helmPath='./helm/' + serviceName + '-canary', + deploymentName=serviceName + '-canary', + ifClause=null, ):: $.deployHelm( $.clusters.prod, deploymentName, { - identifier: "prod", - environment: "prod", + identifier: 'prod', + environment: 'prod', image: { - tag: "deploy-${{ github.sha }}", + tag: 'deploy-${{ github.sha }}', }, replicaCount: 1, } + options, @@ -198,52 +229,57 @@ title='deploy-canary', ifClause=ifClause, ), - helmDeployCanaryJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName + '-canary', - deploymentName = serviceName + '-canary', - image = $.default_job_image, - useCredentials = false, + helmDeployCanaryJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName + '-canary', + deploymentName=serviceName + '-canary', + image=$.default_job_image, + useCredentials=false, ):: - $.ghJob('deploy-canary', + $.ghJob( + 'deploy-canary', image=image, useCredentials=useCredentials, - ifClause='${{ github.event.deployment.environment == \'canary\' }}', + ifClause="${{ github.event.deployment.environment == 'canary' }}", steps=[ $.checkout(), $.helmDeployCanary(serviceName, options, helmPath, deploymentName), ], ), - helmKillCanary(serviceName, - options = {}, - helmPath = './helm/' + serviceName + '-canary', - deploymentName = serviceName + '-canary', - ifClause = null, + helmKillCanary( + serviceName, + options={}, + helmPath='./helm/' + serviceName + '-canary', + deploymentName=serviceName + '-canary', + ifClause=null, ):: $.deployHelm( $.clusters.prod, deploymentName, { - identifier: "prod", - environment: "prod", - image: { - tag: "deploy-${{ github.sha }}", - }, - replicaCount: 0, + identifier: 'prod', + environment: 'prod', + image: { + tag: 'deploy-${{ github.sha }}', + }, + replicaCount: 0, } + options, helmPath, useHelm3=true, title='kill-canary', ifClause=ifClause, ), - helmKillCanaryJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName + '-canary', - deploymentName = serviceName + '-canary', + helmKillCanaryJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName + '-canary', + deploymentName=serviceName + '-canary', ):: - $.ghJob('kill-canary', - ifClause='${{ github.event.deployment.environment == \'kill-canary\' || github.event.deployment.environment == \'production\' }}', + $.ghJob( + 'kill-canary', + ifClause="${{ github.event.deployment.environment == 'kill-canary' || github.event.deployment.environment == 'production' }}", image=$.default_job_image, useCredentials=false, steps=[ diff --git a/.github/jsonnet/images.jsonnet b/.github/jsonnet/images.jsonnet index 659b8a2..fbb9b3d 100644 --- a/.github/jsonnet/images.jsonnet +++ b/.github/jsonnet/images.jsonnet @@ -1,17 +1,17 @@ { - default_rails_step_image: 'eu.gcr.io/unicorn-985/docker-images_ruby-3.1.0_rails-7.0.2.2:deploy-04c4df8affdd378882447beb5b540d063a7e5f6e', - verify_deploy_image: 'eu.gcr.io/unicorn-985/docker-images_ping-service:deploy-82b246eaf567f540757603a9d7bb26e52fc1d803', - jsonnet_bin_image: 'eu.gcr.io/unicorn-985/docker-images_jsonnet:deploy-e6988075021279e18981daa16806105e23ffc8ae', + jsonnet_bin_image: 'eu.gcr.io/unicorn-985/docker-images_jsonnet:v1', helm_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/helm-action:v2', mysql_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/mysql-action:v1', docker_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/push-to-gcr-github-action:v1', - default_job_image: 'alpine:3.16.2', - default_mysql57_image: 'eu.gcr.io/unicorn-985/docker-images_mysql57_utf8mb4:deploy-ce2266ec012075a8077bbd0d7a3dcb2ef05ede68', - default_mysql8_image: 'eu.gcr.io/unicorn-985/docker-images_mysql8_utf8mb4:deploy-e187331bbbc815cc129038442d5b572937e67665', - default_cloudsql_image: 'eu.gcr.io/unicorn-985/docker-images_cloudsql-sidecar:deploy-b8d73f131b54b9877a744b8fab4cb065820d6858', + default_job_image: 'alpine:3.18.3', + default_mysql57_image: 'eu.gcr.io/unicorn-985/docker-images_mysql57_utf8mb4:v1', + default_mysql8_image: 'eu.gcr.io/unicorn-985/docker-images_mysql8_utf8mb4:v1', + default_cloudsql_image: 'eu.gcr.io/unicorn-985/docker-images_cloudsql-sidecar:v1', default_redis_image: 'redis:5.0.6', - default_unicorns_image: 'node:14.20', + default_unicorns_image: 'node:18.15', default_pubsub_image: 'messagebird/gcloud-pubsub-emulator:latest', - default_backend_nest_image: 'node:14.20', - default_mongodb_image: 'mongo:4.2', + default_backend_nest_image: 'node:18.15', + default_mongodb_image: 'eu.gcr.io/unicorn-985/docker-images_mongo6-replicated:v1', + mongo_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/action-mongo-cloner:v1', + default_python_image: 'python:3.11.4', } diff --git a/.github/jsonnet/index.jsonnet b/.github/jsonnet/index.jsonnet index 4775283..c26c847 100644 --- a/.github/jsonnet/index.jsonnet +++ b/.github/jsonnet/index.jsonnet @@ -1,12 +1,18 @@ (import 'base.jsonnet') + -(import 'misc.jsonnet') + -(import 'images.jsonnet') + (import 'clusters.jsonnet') + -(import 'helm.jsonnet') + -(import 'yarn.jsonnet') + +(import 'databases.jsonnet') + (import 'docker.jsonnet') + +(import 'helm.jsonnet') + +(import 'images.jsonnet') + +(import 'misc.jsonnet') + +(import 'mongo.jsonnet') + +(import 'newrelic.jsonnet') + +(import 'pubsub.jsonnet') + +(import 'pulumi.jsonnet') + (import 'ruby.jsonnet') + (import 'services.jsonnet') + -(import 'databases.jsonnet') + -(import 'newrelic.jsonnet') + -(import 'pubsub.jsonnet') +(import 'yarn.jsonnet') + +(import 'deployment.jsonnet') + +(import 'notifications.jsonnet') + +(import 'complete-workflows.jsonnet') + +{} diff --git a/.github/jsonnet/misc.jsonnet b/.github/jsonnet/misc.jsonnet index e6e3542..65372fa 100644 --- a/.github/jsonnet/misc.jsonnet +++ b/.github/jsonnet/misc.jsonnet @@ -1,16 +1,17 @@ { checkout(ifClause=null, fullClone=false, ref=null):: - local with = (if fullClone then { 'fetch-depth': 0 } else {}) + (if ref != null then {'ref': ref } else {}); - $.action('Check out repository code', - 'actions/checkout@v2', - with=with, - ifClause=ifClause + local with = (if fullClone then { 'fetch-depth': 0 } else {}) + (if ref != null then { ref: ref } else {}); + $.action( + 'Check out repository code', + 'actions/checkout@v3', + with=with, + ifClause=ifClause ), lint(service):: $.step('lint-' + service, - './node_modules/.bin/eslint "./packages/' + service + '/{app,lib,tests,config,addon}/**/*.js" --quiet'), - + './node_modules/.bin/eslint "./packages/' + service + '/{app,lib,tests,config,addon}/**/*.js" --quiet'), + lintAll():: $.step('lint', 'yarn lint'), @@ -20,64 +21,58 @@ improvedAudit():: $.step('audit', 'yarn improved-audit'), - verifyJsonnet(fetch_upstream=true):: - $.ghJob('verify-jsonnet-gh-actions', + verifyJsonnet(fetch_upstream=true, runsOn=null):: + $.ghJob( + 'verify-jsonnet-gh-actions', + runsOn=runsOn, image=$.jsonnet_bin_image, steps=[ - $.checkout(ref='${{ github.event.pull_request.head.sha }}'), - $.step('remove-workflows', 'rm -f .github/workflows/*')] + - (if fetch_upstream then [$.step('fetch latest lib-jsonnet', - ' rm -rf .github/jsonnet/; - mkdir .github/jsonnet/; - cd .github; - curl https://files.gynzy.net/lib-jsonnet/v1/jsonnet-prod.tar.gz | tar xvzf -; - ')] else [] - ) - + [$.step('generate-workflows', 'jsonnet -m .github/workflows/ -S .github.jsonnet;'), - $.step('git workaround', 'git config --global --add safe.directory $PWD'), - $.step('check-jsonnet-diff', 'git diff --ignore-space-at-eol --exit-code'), - ], - ), - - verifyDeployJob(name, needs, url, expected_value='${{ github.event.deployment.payload.pr }}', attempts=100):: - $.ghJob(name, - needs=needs, - image = $.verify_deploy_image, - steps=[ - $.step('verify ' + name + ' deploy', - 'bash /ping.sh', - env = { - ATTEMPTS: attempts, - EXPECTED_VALUE: expected_value, - URL: url, - } - ), - ], - runsOn=['ubuntu-latest'], + $.checkout(ref='${{ github.event.pull_request.head.sha }}'), + $.step('remove-workflows', 'rm -f .github/workflows/*'), + ] + + ( + if fetch_upstream then [$.step('fetch latest lib-jsonnet', + ' rm -rf .github/jsonnet/;\n mkdir .github/jsonnet/;\n cd .github;\n curl https://files.gynzy.net/lib-jsonnet/v1/jsonnet-prod.tar.gz | tar xvzf -;\n ')] else [] + ) + + [ + $.step('generate-workflows', 'jsonnet -m .github/workflows/ -S .github.jsonnet;'), + $.step('git workaround', 'git config --global --add safe.directory $PWD'), + $.step('check-jsonnet-diff', 'git diff --exit-code'), + $.step( + 'possible-causes-for-error', + 'echo "Possible causes: \n' + + '1. You updated jsonnet files, but did not regenerate the workflows. \n' + + "To fix, run 'yarn github:generate' locally and commit the changes. If this helps, check if your pre-commit hooks work.\n" + + '2. You used the wrong jsonnet binary. In this case, the newlines at the end of the files differ.\n' + + 'To fix, install the go binary. On mac, run \'brew uninstall jsonnet && brew install jsonnet-go\'"', + ifClause='failure()', + ), + ], ), updatePRDescriptionPipeline( bodyTemplate, - titleTemplate = '', - baseBranchRegex = '[a-z\\d-_.\\\\/]+', - headBranchRegex = '[a-z]+-\\d+', - bodyUpdateAction = 'suffix', - titleUpdateAction = 'prefix', - otherOptions = {}, + titleTemplate='', + baseBranchRegex='[a-z\\d-_.\\\\/]+', + headBranchRegex='[a-z]+-\\d+', + bodyUpdateAction='suffix', + titleUpdateAction='prefix', + otherOptions={}, ):: - $.pipeline('update-pr-description', - event = { - 'pull_request': { types: ['opened'] }, + $.pipeline( + 'update-pr-description', + event={ + pull_request: { types: ['opened'] }, }, - jobs = [ + jobs=[ $.ghJob( 'update-pr-description', - steps = [ + steps=[ $.action( 'update-pr-description', 'gynzy/pr-update-action@v2', with={ - 'repo-token': "${{ secrets.GITHUB_TOKEN }}", + 'repo-token': '${{ secrets.GITHUB_TOKEN }}', 'base-branch-regex': baseBranchRegex, 'head-branch-regex': headBranchRegex, 'title-template': titleTemplate, @@ -85,12 +80,12 @@ 'body-update-action': bodyUpdateAction, 'title-update-action': titleUpdateAction, } + otherOptions, - ) + ), ], useCredentials=false, ), ], - permissions = { + permissions={ 'pull-requests': 'write', }, ), @@ -102,24 +97,151 @@ secret(secretName):: '${{ secrets.' + secretName + ' }}', - pollUrlForContent(url, expectedContent, name='verify-deploy', attempts='100', interval='2000'):: - $.action(name, 'gynzy/wait-for-http-content@v1', + pollUrlForContent(url, expectedContent, name='verify-deploy', attempts='100', interval='2000', ifClause=null):: + $.action( + name, + 'gynzy/wait-for-http-content@v1', with={ url: url, expectedContent: expectedContent, attempts: attempts, interval: interval, - } + }, + ifClause=ifClause, ), - notifiyDeployFailure(channel='#dev-deployments', name='notify-failure', environment="production"):: - $.action(name, 'act10ns/slack@v1', - with={ - status: "${{ job.status }}", - channel: channel, - 'webhook-url': '${{ secrets.SLACK_WEBHOOK_DEPLOY_NOTIFICATION }}', - message: "Deploy of job$ ${{ github.job }} to env: " + environment + " failed!" + cleanupOldBranchesPipelineCron():: + $.pipeline( + 'purge-old-branches', + [ + $.ghJob( + 'purge-old-branches', + useCredentials=false, + steps=[ + $.step('setup', 'apk add git bash'), + $.checkout(), + $.action( + 'Run delete-old-branches-action', + 'beatlabs/delete-old-branches-action@6e94df089372a619c01ae2c2f666bf474f890911', + with={ + repo_token: '${{ github.token }}', + date: '3 months ago', + dry_run: false, + delete_tags: false, + extra_protected_branch_regex: '^(main|master|gynzy|upstream)$', + extra_protected_tag_regex: '^v.*', + exclude_open_pr_branches: true, + }, + env={ + GIT_DISCOVERY_ACROSS_FILESYSTEM: 'true', + } + ), + ], + ), + ], + event={ + schedule: [{ cron: '0 12 * * 1' }], }, - ifClause='failure()', - ) + ), + + codiumAIPRAgent():: + $.pipeline( + 'codium-ai', + [ + $.ghJob( + 'pr_agent_job', + useCredentials=false, + ifClause='${{ github.event.pull_request.draft == false }}', + steps=[ + $.action( + 'PR Agent action step', + 'gynzy/pr-agent@712f0ff0c37b71c676398f73c6ea0198eb9cdd03', + continueOnError=true, + env={ + OPENAI_KEY: $.secret('OPENAI_KEY'), + GITHUB_TOKEN: $.secret('GITHUB_TOKEN'), + }, + ), + ] + ), + ], + event={ + pull_request: { + types: ['opened', 'reopened', 'ready_for_review'], + }, + issue_comment: {}, + } + ), + + // Test if the changed files match the given glob patterns. + // Can test for multiple pattern groups, and sets multiple outputs. + // + // Parameters: + // changedFiles: a map of grouped glob patterns to test against. + // The map key is the name of the group. + // The map value is a list of glob patterns (as string, can use * and **) to test against. + // + // Outputs: + // steps.changes.outputs.: true if the group matched, false otherwise + // + // Permissions: + // Requires the 'pull-requests': 'read' permission + // + // Example: + // $.testForChangedFiles({ + // 'app': ['packages/*/app/**/*', 'package.json'], + // 'lib': ['packages/*/lib/**/*'], + // }) + // + // This will set the following outputs: + // - steps.changes.outputs.app: true if any of the changed files match the patterns in the 'app' group + // - steps.changes.outputs.lib: true if any of the changed files match the patterns in the 'lib' group + // + // These can be tested as in an if clause as follows: + // if: steps.changes.outputs.app == 'true' + // + // See https://github.com/dorny/paths-filter for more information. + testForChangedFiles(changedFiles):: + [ + $.step('git safe directory', 'git config --global --add safe.directory $PWD'), + $.action( + 'check-for-changes', + uses='dorny/paths-filter@v2', + id='changes', + with={ + filters: ||| + %s + ||| % std.manifestYamlDoc(changedFiles), + token: '${{ github.token }}', + } + ), + ], + + awaitJob(name, jobs):: + local dependingJobs = std.flatMap( + function(job) + local jobNameArray = std.objectFields(job); + if std.length(jobNameArray) == 1 then [jobNameArray[0]] else [], + jobs + ); + [ + $.ghJob( + 'await-' + name, + ifClause='${{ always() }}', + needs=dependingJobs, + useCredentials=false, + steps=[ + $.step( + 'success', + 'exit 0', + ifClause="${{ contains(join(needs.*.result, ','), 'success') }}" + ), + $.step( + 'failure', + 'exit 1', + ifClause="${{ contains(join(needs.*.result, ','), 'failure') }}" + ), + ], + ), + ], } diff --git a/.github/jsonnet/mongo.jsonnet b/.github/jsonnet/mongo.jsonnet new file mode 100644 index 0000000..6e3a933 --- /dev/null +++ b/.github/jsonnet/mongo.jsonnet @@ -0,0 +1,65 @@ +{ + mongo_servers: { + test: { + type: 'mongo', + name: 'test', + connectionString: 'test-pri.kl1s6.gcp.mongodb.net', + gke_cluster: 'test', + gke_project: 'gynzy-test-project', + gke_zone: 'europe-west4-b', + password_secret: 'mongodb-pass-test', + gce_json: $.secret('gce_new_test_json'), + lifecycle: 'test', + projectId: '5da5889579358e19bf4b16ea' // test + }, + + prod: { + type: 'mongo', + name: 'prod', + connectionString: 'prod-pri.silfd.gcp.mongodb.net', + projectId: '5dde7f71a6f239a82fa155f4', // prod + }, + 'board-prod': { + type: 'mongo', + name: 'board-prod', + connectionString: 'board-prod-pri.silfd.mongodb.net', + projectId: '5dde7f71a6f239a82fa155f4', // prod + }, + 'adaptive-learning-prod': { + type: 'mongo', + name: 'adaptive-learning-prod', + connectionString: 'adaptive-learning-prod-pri.silfd.mongodb.net', + projectId: '5dde7f71a6f239a82fa155f4', // prod + }, + 'accounts-prod': { + type: 'mongo', + name: 'accounts-prod', + connectionString: 'accounts-prod-pri.silfd.mongodb.net', + projectId: '5dde7f71a6f239a82fa155f4', // prod + }, + 'interaction-prod': { + type: 'mongo', + name: 'interaction-prod', + connectionString: 'interaction-prod-pri.silfd.mongodb.net', + projectId: '5dde7f71a6f239a82fa155f4', // prod + }, + }, + + copyMongoDatabase(mongoActionsOptions):: + assert std.length(std.findSubstr('_pr_', mongoActionsOptions.MONGO_DST)) > 0; // target db gets deleted. must contain _pr_ + + $.action( + 'copy-mongo-db', + $.mongo_action_image, + env=mongoActionsOptions { TASK: 'clone' } + ), + + deleteMongoPrDatabase(mongoActionsOptions):: + assert std.length(std.findSubstr('_pr_', mongoActionsOptions.MONGO_DST)) > 0; // target db gets deleted. must contain _pr_ + + $.action( + 'delete-mongo-db', + $.mongo_action_image, + env=mongoActionsOptions { TASK: 'delete' } + ), +} diff --git a/.github/jsonnet/newrelic.jsonnet b/.github/jsonnet/newrelic.jsonnet index 7a94a76..d01dccf 100644 --- a/.github/jsonnet/newrelic.jsonnet +++ b/.github/jsonnet/newrelic.jsonnet @@ -1,29 +1,32 @@ { - postReleaseToNewRelicJob( - apps, + postReleaseToNewRelicJob( + apps, - ):: - $.ghJob('post-newrelic-release', + ):: + $.ghJob( + 'post-newrelic-release', image=$.default_backend_nest_image, useCredentials=false, ifClause="${{ github.event.deployment.environment == 'production' }}", steps=[ $.checkoutAndYarn(ref='${{ github.sha }}'), - $.step('post-newrelic-release', + $.step( + 'post-newrelic-release', 'node .github/scripts/newrelic.js', env={ NEWRELIC_API_KEY: $.secret('NEWRELIC_API_KEY'), - NEWRELIC_APPS: std.join(" ", std.flatMap( - function (app) - if std.objectHas(app, 'newrelicApps') then - app.newrelicApps else [], - apps) + NEWRELIC_APPS: std.join( + ' ', std.flatMap( + function(app) + if std.objectHas(app, 'newrelicApps') then + app.newrelicApps else [], + apps + ) ), GIT_COMMIT: '${{ github.sha }}', - DRONE_SOURCE_BRANCH: '${{ github.ref_name }}', - + DRONE_SOURCE_BRANCH: '${{ github.event.deployment.payload.branch }}', } - ) + ), ], - ) + ), } diff --git a/.github/jsonnet/notifications.jsonnet b/.github/jsonnet/notifications.jsonnet new file mode 100644 index 0000000..6cb0e7d --- /dev/null +++ b/.github/jsonnet/notifications.jsonnet @@ -0,0 +1,27 @@ +{ + notifiyDeployFailure(channel='#dev-deployments', name='notify-failure', environment='production'):: + $.action( + name, + 'act10ns/slack@v2', + with={ + status: '${{ job.status }}', + channel: channel, + 'webhook-url': '${{ secrets.SLACK_WEBHOOK_DEPLOY_NOTIFICATION }}', + message: 'Deploy of job ${{ github.job }} to env: ' + environment + ' failed!', + }, + ifClause='failure()', + ), + + sendSlackMessage(channel='#dev-deployments', stepName='sendSlackMessage', message=null, ifClause=null):: + $.action( + stepName, + 'act10ns/slack@v2', + with={ + status: 'starting', + channel: channel, + 'webhook-url': '${{ secrets.SLACK_WEBHOOK_DEPLOY_NOTIFICATION }}', + message: message, + }, + ifClause=ifClause, + ), +} diff --git a/.github/jsonnet/pubsub.jsonnet b/.github/jsonnet/pubsub.jsonnet index 4afb354..702be96 100644 --- a/.github/jsonnet/pubsub.jsonnet +++ b/.github/jsonnet/pubsub.jsonnet @@ -1,18 +1,19 @@ { deletePrPubsubSubscribersJob(needs=null):: - $.ghJob('delete-pubsub-pr-subscribers', - useCredentials=false, - image='google/cloud-sdk:alpine', - steps=[ - $.configureGoogleAuth($.secret('GCE_NEW_TEST_JSON')), - $.step('install jq', 'apk add jq'), - $.step('show auth', 'gcloud auth list'), - $.step('wait for pod termination', 'sleep 60'), - $.step('delete subscriptions', ' - gcloud --project gynzy-test-project pubsub subscriptions list --format json | jq -r \'.[].name\' | grep -- \'-pr-${{ github.event.number }}\' | xargs -r gcloud --project gynzy-test-project pubsub subscriptions delete' - ), - ], - needs=needs, - ) + $.ghJob( + 'delete-pubsub-pr-subscribers', + useCredentials=false, + image='google/cloud-sdk:alpine', + steps=[ + $.configureGoogleAuth($.secret('GCE_NEW_TEST_JSON')), + $.step('install jq', 'apk add jq'), + $.step('show auth', 'gcloud auth list'), + $.step('wait for pod termination', 'sleep 60'), + $.step( + 'delete subscriptions', "\n gcloud --project gynzy-test-project pubsub subscriptions list --format json | jq -r '.[].name' | grep -- '-pr-${{ github.event.number }}' | xargs -r gcloud --project gynzy-test-project pubsub subscriptions delete" + ), + ], + needs=needs, + ), -} \ No newline at end of file +} diff --git a/.github/jsonnet/pull-upstream-and-rebuild.sh b/.github/jsonnet/pull-upstream-and-rebuild.sh index 3de0494..f6474ab 100644 --- a/.github/jsonnet/pull-upstream-and-rebuild.sh +++ b/.github/jsonnet/pull-upstream-and-rebuild.sh @@ -1,6 +1,6 @@ #!/bin/sh # Use environment PR_NUMBER to fetch a pull request. -# export PR_NUMBER=12345; sh .github/jsonnet/pull-upstream-and-rebuild.sh +# export PR_NUMBER=12345 sh .github/jsonnet/pull-upstream-and-rebuild.sh # # Usage: sh .github/jsonnet/pull-upstream-and-rebuild.sh diff --git a/.github/jsonnet/pulumi.jsonnet b/.github/jsonnet/pulumi.jsonnet new file mode 100644 index 0000000..035061e --- /dev/null +++ b/.github/jsonnet/pulumi.jsonnet @@ -0,0 +1,154 @@ +{ + local pulumiSetupSteps = [ + $.action( + 'auth', + uses='google-github-actions/auth@v1', + id='auth', + with={ + credentials_json: $.secret('PULUMI_SERVICE_ACCOUNT'), + } + ), + $.action('setup-gcloud', uses='google-github-actions/setup-gcloud@v0'), + $.action('pulumi-cli-setup', 'pulumi/actions@v4'), + ], + + pulumiPreview( + stack, + pulumiDir=null, + ): $.action( + 'pulumi-preview-' + stack, + uses='pulumi/actions@v4', + with={ + command: 'preview', + 'stack-name': stack, + 'work-dir': pulumiDir, + 'comment-on-pr': true, + 'github-token': '${{ secrets.GITHUB_TOKEN }}', + upsert: true, + refresh: true, + }, + env={ + PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', + } + ), + + pulumiPreviewJob( + stack, + pulumiDir=null, + yarnDir=null, + gitCloneRef='${{ github.event.pull_request.head.sha }}', + cacheName=null, + ): $.ghJob( + 'pulumi-preview-' + stack, + image='node:18', + useCredentials=false, + steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + + pulumiSetupSteps + + [$.pulumiPreview(stack, pulumiDir=pulumiDir)], + ), + + pulumiPreviewTestJob( + stack='test', + pulumiDir=null, + yarnDir=null, + gitCloneRef='${{ github.event.pull_request.head.sha }}', + cacheName=null, + ): $.pulumiPreviewJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName), + + pulumiPreviewProdJob( + stack='prod', + pulumiDir=null, + yarnDir=null, + gitCloneRef='${{ github.event.pull_request.head.sha }}', + cacheName=null, + ): $.pulumiPreviewJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName), + + pulumiPreviewTestAndProdJob( + pulumiDir=null, + yarnDir=null, + gitCloneRef='${{ github.event.pull_request.head.sha }}', + cacheName=null, + ): $.ghJob( + 'pulumi-preview', + image='node:18', + useCredentials=false, + steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + + pulumiSetupSteps + + [ + $.pulumiPreview('test', pulumiDir=pulumiDir), + $.pulumiPreview('prod', pulumiDir=pulumiDir), + ], + ), + + pulumiDeployJob( + stack, + pulumiDir=null, + yarnDir=null, + gitCloneRef='${{ github.sha }}', + cacheName=null, + ifClause=null, + ): $.ghJob( + 'pulumi-deploy-' + stack, + ifClause=ifClause, + image='node:18', + useCredentials=false, + steps=[$.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir)] + + pulumiSetupSteps + + [ + $.action( + 'pulumi-deploy-' + stack, + uses='pulumi/actions@v4', + with={ + command: 'up', + 'stack-name': stack, + 'work-dir': pulumiDir, + upsert: true, + refresh: true, + }, + env={ + PULUMI_CONFIG_PASSPHRASE: '${{ secrets.PULUMI_CONFIG_PASSPHRASE }}', + } + ), + $.notifiyDeployFailure(), + ], + ), + + pulumiDeployTestJob( + stack='test', + pulumiDir=null, + yarnDir=null, + gitCloneRef='${{ github.sha }}', + cacheName=null, + ifClause="${{ github.event.deployment.environment == 'test' }}", + ): $.pulumiDeployJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName, ifClause=ifClause), + + pulumiDeployProdJob( + stack='prod', + pulumiDir=null, + yarnDir=null, + gitCloneRef='${{ github.sha }}', + cacheName=null, + ifClause="${{ github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}", + ): $.pulumiDeployJob(stack, pulumiDir=pulumiDir, yarnDir=yarnDir, gitCloneRef=gitCloneRef, cacheName=cacheName, ifClause=ifClause), + + pulumiDefaultPipeline( + pulumiDir='.', + yarnDir=null, + cacheName=null, + deployTestWithProd=false, + ): + $.pipeline( + 'pulumi-preview', + [ + $.pulumiPreviewTestAndProdJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName), + ], + ) + + $.pipeline( + 'pulumi-deploy', + [ + $.pulumiDeployTestJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName, ifClause=if deployTestWithProd then "${{ github.event.deployment.environment == 'test' || github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}" else "${{ github.event.deployment.environment == 'test' }}"), + $.pulumiDeployProdJob(pulumiDir=pulumiDir, yarnDir=yarnDir, cacheName=cacheName), + ], + event='deployment', + ), +} diff --git a/.github/jsonnet/ruby.jsonnet b/.github/jsonnet/ruby.jsonnet index d6878f7..bafda85 100644 --- a/.github/jsonnet/ruby.jsonnet +++ b/.github/jsonnet/ruby.jsonnet @@ -1,105 +1,115 @@ { - rubyDeployPRPipeline(serviceName, - dockerImageName = 'github-gynzy-docker-' + serviceName, - helmDeployOptions = { + rubyDeployPRPipeline( + serviceName, + dockerImageName='github-gynzy-docker-' + serviceName, + helmDeployOptions={ ingress: { enabled: true }, cronjob: { enabled: true }, }, - mysqlCloneOptions = {}, - migrateOptions = {} + mysqlCloneOptions={}, + migrateOptions={}, + rubyImageName=null, ):: + assert rubyImageName != null; local mysqlCloneOptionsWithDefaults = { - enabled: false, // default for backwards compatibility. example params below - database_name_target: serviceName + '_pr_${{ github.event.number }}', - database_name_source: serviceName, - database_host: 'cloudsql-proxy', - database_username: serviceName, - database_password: $.secret('database_password_test'), + enabled: false, // default for backwards compatibility. example params below + database_name_target: serviceName + '_pr_${{ github.event.number }}', + database_name_source: serviceName, + database_host: 'cloudsql-proxy', + database_username: serviceName, + database_password: $.secret('database_password_test'), } + mysqlCloneOptions; local migrateOptionsWithDefaults = { - enabled: false, - RAILS_ENV: "production", - RAILS_DB_HOST: "cloudsql-proxy", - RAILS_DB_NAME: serviceName + "_pr_${{ github.event.number }}", - RAILS_DB_PASSWORD: $.secret('database_password_test'), - RAILS_DB_USER: serviceName, - } + migrateOptions; + enabled: false, + RAILS_ENV: 'production', + RAILS_DB_HOST: 'cloudsql-proxy', + RAILS_DB_NAME: serviceName + '_pr_${{ github.event.number }}', + RAILS_DB_PASSWORD: $.secret('database_password_test'), + RAILS_DB_USER: serviceName, + } + migrateOptions; - $.pipeline('deploy-pr', + $.pipeline( + 'deploy-pr', [ - $.ghJob('deploy-pr', - image = $.default_rails_step_image, - steps = [ - $.checkout(ref='${{ github.event.pull_request.head.sha }}'), - $.setVerionFile()] + - (if mysqlCloneOptionsWithDefaults.enabled then [$.copyDatabase(mysqlCloneOptionsWithDefaults)] else []) + - (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + - [ - $.buildDocker(dockerImageName, - env = { - BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), - }, - build_args = 'BUNDLE_GITHUB__COM=' + $.secret('BUNDLE_GITHUB__COM'), - ), - $.helmDeployPR(serviceName, helmDeployOptions), - ], - services = {} + - (if mysqlCloneOptionsWithDefaults.enabled then {'cloudsql-proxy': $.cloudsql_proxy_service(mysqlCloneOptionsWithDefaults.database)} else {}) - ) + $.ghJob( + 'deploy-pr', + image=rubyImageName, + steps=[ + $.checkout(ref='${{ github.event.pull_request.head.sha }}'), + $.setVerionFile(), + ] + + (if mysqlCloneOptionsWithDefaults.enabled then [$.copyDatabase(mysqlCloneOptionsWithDefaults)] else []) + + (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + + [ + $.buildDocker( + dockerImageName, + env={ + BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), + }, + build_args='BUNDLE_GITHUB__COM=' + $.secret('BUNDLE_GITHUB__COM'), + ), + $.helmDeployPR(serviceName, helmDeployOptions), + ], + services={} + + (if mysqlCloneOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(mysqlCloneOptionsWithDefaults.database) } else {}) + ), ], event='pull_request', ), rubyMigrate(migrateOptions):: - local env = { - BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), - SSO_PUBLIC_KEY: '', - RAILS_ENV: "production", - RAILS_DB_HOST: migrateOptions.RAILS_DB_HOST, - RAILS_DB_NAME: migrateOptions.RAILS_DB_NAME, - RAILS_DB_PASSWORD: migrateOptions.RAILS_DB_PASSWORD, - RAILS_DB_USER: migrateOptions.RAILS_DB_USER, - }; + local env = { + BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), + SSO_PUBLIC_KEY: '', + RAILS_ENV: 'production', + RAILS_DB_HOST: migrateOptions.RAILS_DB_HOST, + RAILS_DB_NAME: migrateOptions.RAILS_DB_NAME, + RAILS_DB_PASSWORD: migrateOptions.RAILS_DB_PASSWORD, + RAILS_DB_USER: migrateOptions.RAILS_DB_USER, + }; - [ - $.step('bundle install', 'bundle install', env={BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM')}), - $.step('migrate-db', 'rails db:migrate;', env=env ) - ] - , + [ + $.step('bundle install', 'bundle install', env={ BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM') }), + $.step('migrate-db', 'rails db:migrate;', env=env), + ] + , - deployApiDocs(serviceName, + deployApiDocs( + serviceName, enableDatabase=false, generateCommands=null, extra_env={}, - services={ 'db': $.mysql57service(database='ci', password='ci', root_password='1234test', username='ci') } + services={ db: $.mysql57service(database='ci', password='ci', root_password='1234test', username='ci') }, + rubyImageName=null, ):: - $.ghJob('apidocs', - image = $.default_rails_step_image, - ifClause = '${{ github.event.deployment.environment == \'production\' }}', + assert rubyImageName != null; + $.ghJob( + 'apidocs', + image=rubyImageName, + ifClause="${{ github.event.deployment.environment == 'production' }}", steps=[ $.checkout(), - $.step('generate', + $.step( + 'generate', (if generateCommands != null then generateCommands else - ' bundle config --delete without; - bundle install; - bundle exec rails db:test:prepare; - bundle exec rails docs:generate; - '), + ' bundle config --delete without;\n bundle install;\n bundle exec rails db:test:prepare;\n bundle exec rails docs:generate;\n '), env={ - RAILS_ENV: 'test', - GOOGLE_PRIVATE_KEY: $.secret('GOOGLE_PRIVATE_KEY'), - BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), - } + - (if enableDatabase then - { - RAILS_DB_HOST: 'db', - RAILS_DB_NAME: 'ci', - RAILS_DB_PASSWORD: 'ci', - RAILS_DB_USER: 'ci', - } else {}) + extra_env + RAILS_ENV: 'test', + GOOGLE_PRIVATE_KEY: $.secret('GOOGLE_PRIVATE_KEY'), + BUNDLE_GITHUB__COM: $.secret('BUNDLE_GITHUB__COM'), + } + + (if enableDatabase then + { + RAILS_DB_HOST: 'db', + RAILS_DB_NAME: 'ci', + RAILS_DB_PASSWORD: 'ci', + RAILS_DB_USER: 'ci', + } else {}) + extra_env ), - $.action('setup auth', 'google-github-actions/auth@v0', + $.action( + 'setup auth', + 'google-github-actions/auth@v1', with={ credentials_json: $.secret('GCE_JSON'), }, @@ -111,20 +121,18 @@ services=(if enableDatabase then services else null), ), - setVerionFile(version = '${{ github.event.pull_request.head.sha }}', file = 'VERSION'):: + setVerionFile(version='${{ github.event.pull_request.head.sha }}', file='VERSION'):: $.step( 'set-version', - 'echo "' + version + '" > ' + file + '; - echo "Generated version number:"; - cat ' + file + '; - ' + 'echo "' + version + '" > ' + file + ';\n echo "Generated version number:";\n cat ' + file + ';\n ' ), - rubyDeletePRPipeline(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-pr-${{ github.event.number }}', - mysqlDeleteOptions = {}, + rubyDeletePRPipeline( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-pr-${{ github.event.number }}', + mysqlDeleteOptions={}, ):: local mysqlDeleteOptionsWithDefaults = { enabled: false, @@ -134,72 +142,79 @@ database_password: $.secret('database_password_test'), } + mysqlDeleteOptions; - $.pipeline('close-pr', + $.pipeline( + 'close-pr', [ $.helmDeletePRJob(serviceName, options, helmPath, deploymentName, mysqlDeleteOptionsWithDefaults), ], - event = { + event={ pull_request: { - types: ['closed'] - } + types: ['closed'], + }, }, ), - rubyDeployTestJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-master', - image = $.default_job_image, - useCredentials = false, - migrateOptions = {}, + rubyDeployTestJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-master', + image=null, + useCredentials=false, + migrateOptions={}, ):: + assert image != null; local migrateOptionsWithDefaults = { enabled: false, - RAILS_ENV: "production", - RAILS_DB_HOST: "cloudsql-proxy", + RAILS_ENV: 'production', + RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName, RAILS_DB_PASSWORD: $.secret('database_password_test'), RAILS_DB_USER: serviceName, } + migrateOptions; - $.ghJob('deploy-test', - ifClause='${{ github.event.deployment.environment == \'test\' }}', + $.ghJob( + 'deploy-test', + ifClause="${{ github.event.deployment.environment == 'test' }}", image=image, - useCredentials = useCredentials, + useCredentials=useCredentials, steps= - [$.checkout()] + - (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + - [$.helmDeployTest(serviceName, options, helmPath, deploymentName)], - services = {} + - (if migrateOptionsWithDefaults.enabled then {'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database)} else {}) + [$.checkout()] + + (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + + [$.helmDeployTest(serviceName, options, helmPath, deploymentName)], + services={} + + (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) ), - rubyDeployProdJob(serviceName, - options = {}, - helmPath = './helm/' + serviceName, - deploymentName = serviceName + '-prod', - image = $.default_job_image, - useCredentials = false, - migrateOptions = {}, + rubyDeployProdJob( + serviceName, + options={}, + helmPath='./helm/' + serviceName, + deploymentName=serviceName + '-prod', + image=null, + useCredentials=false, + migrateOptions={}, ):: + assert image != null; local migrateOptionsWithDefaults = { enabled: false, - RAILS_ENV: "production", - RAILS_DB_HOST: "cloudsql-proxy", + RAILS_ENV: 'production', + RAILS_DB_HOST: 'cloudsql-proxy', RAILS_DB_NAME: serviceName, RAILS_DB_PASSWORD: $.secret('database_password_production'), RAILS_DB_USER: serviceName, } + migrateOptions; - $.ghJob('deploy-prod', - ifClause='${{ github.event.deployment.environment == \'production\' }}', + $.ghJob( + 'deploy-prod', + ifClause="${{ github.event.deployment.environment == 'production' }}", image=image, useCredentials=useCredentials, steps=[$.checkout()] + - (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + - [$.helmDeployProd(serviceName, options, helmPath, deploymentName)] + [ $.notifiyDeployFailure() ], - services = {} + - (if migrateOptionsWithDefaults.enabled then {'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database)} else {}) + (if migrateOptionsWithDefaults.enabled then $.rubyMigrate(migrateOptionsWithDefaults) else []) + + [$.helmDeployProd(serviceName, options, helmPath, deploymentName)] + [$.notifiyDeployFailure()], + services={} + + (if migrateOptionsWithDefaults.enabled then { 'cloudsql-proxy': $.cloudsql_proxy_service(migrateOptionsWithDefaults.database) } else {}) ), diff --git a/.github/jsonnet/services.jsonnet b/.github/jsonnet/services.jsonnet index 550d706..35fbb63 100644 --- a/.github/jsonnet/services.jsonnet +++ b/.github/jsonnet/services.jsonnet @@ -1,55 +1,55 @@ { mysql57service(database=null, password=null, root_password=null, username=null, port='3306'):: - { - image: $.default_mysql57_image, - credentials: { - username: '_json_key', - password: $.secret('docker_gcr_io'), + { + image: $.default_mysql57_image, + credentials: { + username: '_json_key', + password: $.secret('docker_gcr_io'), + }, + env: { + MYSQL_DATABASE: database, + MYSQL_PASSWORD: password, + MYSQL_ROOT_PASSWORD: root_password, + MYSQL_USER: username, + MYSQL_TCP_PORT: port, + }, + options: '--health-cmd="mysqladmin ping" --health-interval=1s --health-timeout=1s --health-retries=40', + ports: [port + ':' + port], }, - env: { - MYSQL_DATABASE: database, - MYSQL_PASSWORD: password, - MYSQL_ROOT_PASSWORD: root_password, - MYSQL_USER: username, - MYSQL_TCP_PORT: port, - }, - options: '--health-cmd="mysqladmin ping" --health-interval=1s --health-timeout=1s --health-retries=40', - ports: [port + ':' + port], - }, mysql8service(database=null, password=null, root_password=null, username=null, port='3306'):: - { - image: $.default_mysql8_image, - credentials: { - username: '_json_key', - password: $.secret('docker_gcr_io'), + { + image: $.default_mysql8_image, + credentials: { + username: '_json_key', + password: $.secret('docker_gcr_io'), + }, + env: { + MYSQL_DATABASE: database, + MYSQL_PASSWORD: password, + MYSQL_ROOT_PASSWORD: root_password, + MYSQL_USER: username, + MYSQL_TCP_PORT: port, + }, + options: '--health-cmd="mysqladmin ping" --health-interval=1s --health-timeout=1s --health-retries=40', + ports: [port + ':' + port], }, - env: { - MYSQL_DATABASE: database, - MYSQL_PASSWORD: password, - MYSQL_ROOT_PASSWORD: root_password, - MYSQL_USER: username, - MYSQL_TCP_PORT: port, - }, - options: '--health-cmd="mysqladmin ping" --health-interval=1s --health-timeout=1s --health-retries=40', - ports: [port + ':' + port], - }, cloudsql_proxy_service(database):: - { - image: $.default_cloudsql_image, - credentials: { - username: '_json_key', - password: $.secret('docker_gcr_io'), + { + image: $.default_cloudsql_image, + credentials: { + username: '_json_key', + password: $.secret('docker_gcr_io'), + }, + env: { + GOOGLE_PROJECT: database.project, + CLOUDSQL_ZONE: database.region, + CLOUDSQL_INSTANCE: database.server, + SERVICE_JSON: $.secret('GCE_JSON'), + }, + ports: ['3306:3306'], }, - env: { - GOOGLE_PROJECT: database.project, - CLOUDSQL_ZONE: database.region, - CLOUDSQL_INSTANCE: database.server, - SERVICE_JSON: $.secret('GCE_JSON'), - }, - ports: ['3306:3306'], - }, redis_service():: { image: $.default_redis_image, @@ -62,19 +62,23 @@ }, serviceMongodb( - service, - name = 'mongodb-' + service, - username = 'root', - password = 'therootpass', - image = $.default_mongodb_image, + service, + name='mongodb-' + service, + username='root', + password='therootpass', ):: { - [name]: { - image: image, - ports: ['27017:27017'], - env: { - MONGO_INITDB_ROOT_USERNAME: username, - MONGO_INITDB_ROOT_PASSWORD: password, - } - } - } -} \ No newline at end of file + [name]: { + image: $.default_mongodb_image, + ports: ['27017:27017'], + credentials: { + username: '_json_key', + password: $.secret('docker_gcr_io'), + }, + env: { + MONGO_INITDB_ROOT_USERNAME: username, + MONGO_INITDB_ROOT_PASSWORD: password, + MONGO_REPLICA_SET_NAME: 'rs0', + }, + }, + }, +} diff --git a/.github/jsonnet/yarn.jsonnet b/.github/jsonnet/yarn.jsonnet index 41713dd..172541d 100644 --- a/.github/jsonnet/yarn.jsonnet +++ b/.github/jsonnet/yarn.jsonnet @@ -1,45 +1,78 @@ { - yarn(ifClause=null, prod=false):: + yarn(ifClause=null, prod=false, workingDirectory=null):: $.step( 'yarn' + (if prod then '-prod' else ''), run='yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline' + (if prod then ' --prod' else '') + ' || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline' + (if prod then ' --prod' else ''), ifClause=ifClause, + workingDirectory=workingDirectory ), - setNpmToken(ifClause=null):: - $.step('set npm_token', run=' -cat < .npmrc -registry=https://npm.gynzy.net/ -always-auth="true" -"//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" -EOF - ', + setNpmToken(ifClause=null, workingDirectory=null):: $.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory), + + setGynzyNpmToken(ifClause=null, workingDirectory=null):: + $.step( + 'set gynzy npm_token', + run= + ||| + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + |||, env={ NPM_TOKEN: $.secret('npm_token'), }, ifClause=ifClause, + workingDirectory=workingDirectory, ), - checkoutAndYarn(cacheName=null, ifClause=null, fullClone=false, ref=null, prod=false):: + setGithubNpmToken(ifClause=null, workingDirectory=null):: + $.step( + 'set github npm_token', + run= + ||| + cat < .npmrc + @gynzy:registry=https://npm.pkg.github.com + always-auth=true + //npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN} + EOF + |||, + env={ + NODE_AUTH_TOKEN: $.secret('GITHUB_TOKEN'), + }, + ifClause=ifClause, + workingDirectory=workingDirectory, + ), + + checkoutAndYarn(cacheName=null, ifClause=null, fullClone=false, ref=null, prod=false, workingDirectory=null):: $.checkout(ifClause=ifClause, fullClone=fullClone, ref=ref) + - $.setNpmToken(ifClause=ifClause) + - (if cacheName == null then [] else $.fetchYarnCache(cacheName, ifClause=ifClause)) + - $.yarn(ifClause=ifClause, prod=prod), + $.setGynzyNpmToken(ifClause=ifClause, workingDirectory=workingDirectory) + + (if cacheName == null then [] else $.fetchYarnCache(cacheName, ifClause=ifClause, workingDirectory=workingDirectory)) + + $.yarn(ifClause=ifClause, prod=prod, workingDirectory=workingDirectory), - fetchYarnCache(cacheName, ifClause=null):: $.step( + fetchYarnCache(cacheName, ifClause=null, workingDirectory=null):: $.step( 'download yarn cache', - run='echo "Downloading yarn cache && node_modules"\nwget -q -O - https://storage.googleapis.com/files-gynzy-com-test/yarn-cache/' + cacheName + '.tar.gz | tar xfz - -if [ $? -ne 0 ]; then - # download failed. cleanup node_modules because it can contain a yarn integrity file but not have all the data as specified - echo "Cache download failed, cleanup up caches and run yarn without cache" - find . -type d -name \'node_modules\' | xargs rm -rf - rm -rf .yarncache - echo "Cleanup complete" -else - echo "Finished downloading yarn cache && node_modules" -fi -', - ifClause=ifClause + run= + ||| + set +e; + echo "Downloading yarn cache && node_modules" + wget -q -O - "https://storage.googleapis.com/files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz" | tar xfz - + if [ $? -ne 0 ]; then + # download failed. cleanup node_modules because it can contain a yarn integrity file but not have all the data as specified + echo "Cache download failed, cleanup up caches and run yarn without cache" + find . -type d -name 'node_modules' | xargs rm -rf + rm -rf .yarncache + echo "Cleanup complete" + else + echo "Finished downloading yarn cache && node_modules" + fi + |||, + ifClause=ifClause, + workingDirectory=workingDirectory, + env={ + CACHE_NAME: cacheName, + }, ), updateYarnCachePipeline(cacheName, appsDir='packages', image=null, useCredentials=null):: @@ -53,31 +86,36 @@ fi ifClause="${{ github.event.deployment.environment == 'production' || github.event.deployment.environment == 'prod' }}", steps=[ $.checkout() + - $.setNpmToken() + + $.setGynzyNpmToken() + $.yarn(), $.action( 'setup auth', - 'google-github-actions/auth@v0', + 'google-github-actions/auth@v1', with={ credentials_json: $.secret('SERVICE_JSON'), }, id='auth', ), $.action('setup-gcloud', 'google-github-actions/setup-gcloud@v0'), - $.step('upload-yarn-cache', - ' -set -e + $.step( + 'upload-yarn-cache', + ||| + set -e -echo "Creating cache archive" -# v1 -ls ' + appsDir + '/*/node_modules -1 -d 2>/dev/null | xargs tar -czf ' + cacheName + '.tar.gz .yarncache node_modules + echo "Creating cache archive" + # v1 + ls "${APPS_DIR}/*/node_modules" -1 -d 2>/dev/null | xargs tar -czf "${CACHE_NAME}.tar.gz" .yarncache node_modules -echo "Upload cache" -gsutil cp ' + cacheName + '.tar.gz gs://files-gynzy-com-test/yarn-cache/' + cacheName + '.tar.gz.tmp -gsutil mv gs://files-gynzy-com-test/yarn-cache/' + cacheName + '.tar.gz.tmp gs://files-gynzy-com-test/yarn-cache/' + cacheName + '.tar.gz + echo "Upload cache" + gsutil cp "${CACHE_NAME}.tar.gz" "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz.tmp" + gsutil mv "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz.tmp" "gs://files-gynzy-com-test/yarn-cache/${CACHE_NAME}.tar.gz" -echo "Upload finished" -', + echo "Upload finished" + |||, + env={ + CACHE_NAME: cacheName, + APPS_DIR: appsDir, + } ), ], ), @@ -87,36 +125,113 @@ echo "Upload finished" configureGoogleAuth(secret):: $.step( 'activate google service account', - run='printf \'%s\' "$SERVICE_JSON" > /gce.json; - gcloud auth activate-service-account --key-file=/gce.json; - gcloud --quiet auth configure-docker', + run= + ||| + printf '%s' "${SERVICE_JSON}" > gce.json; + gcloud auth activate-service-account --key-file=gce.json; + gcloud --quiet auth configure-docker; + rm gce.json + |||, env={ SERVICE_JSON: secret }, ), - yarnPublish(isPr=true):: - $.step('publish', - ' -bash -c \'set -xeo pipefail; - -VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P \'[0-9a-zA-Z_.-]+$\' ); -if [[ ! -z "${PR_NUMBER}" ]]; then - echo "Setting tag/version for pr build."; - TAG=pr-$PR_NUMBER; - PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; -elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then - if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then - echo "Tag version does not match package version. They should match. Exiting"; - exit 1; - fi - echo "Setting tag/version for release/tag build."; - PUBLISHVERSION=$VERSION; - TAG="latest"; -else - exit 1 -fi - -yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"\'; - ', - env={} + (if isPr then { PR_NUMBER: '${{ github.event.number }}' } else {})), + yarnPublish(isPr=true, ifClause=null):: + $.step( + 'publish', + ||| + bash -c 'set -xeo pipefail; + + cp package.json package.json.bak; + + VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); + if [[ ! -z "${PR_NUMBER}" ]]; then + echo "Setting tag/version for pr build."; + TAG=pr-$PR_NUMBER; + PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then + echo "Tag version does not match package version. They should match. Exiting"; + exit 1; + fi + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + elif [[ "${GITHUB_REF_TYPE}" == "branch" && "${GITHUB_REF_NAME}" == "main" ]]; then + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + else + exit 1 + fi + yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; + + mv package.json.bak package.json; + '; + |||, + env={} + (if isPr then { PR_NUMBER: '${{ github.event.number }}' } else {}), + ifClause=ifClause, + ), + + yarnPublishToRepositories(isPr, repositories, ifClause=null):: + (std.flatMap(function(repository) + if repository == 'gynzy' then [$.setGynzyNpmToken(ifClause=ifClause), $.yarnPublish(isPr=isPr, ifClause=ifClause)] + else if repository == 'github' then [$.setGithubNpmToken(ifClause=ifClause), $.yarnPublish(isPr=isPr, ifClause=ifClause)] + else error 'Unknown repository type given.', + repositories)), + + + yarnPublishPreviewJob( + image='node:18', + useCredentials=false, + gitCloneRef='${{ github.event.pull_request.head.sha }}', + buildSteps=[$.step('build', 'yarn build')], + checkVersionBump=true, + repositories=['gynzy'], + onChangedFiles=false, + runsOn=null, + ): + local ifClause = (if onChangedFiles != false then "steps.changes.outputs.package == 'true'" else null); + $.ghJob( + 'yarn-publish-preview', + runsOn=runsOn, + image='node:18', + useCredentials=false, + steps= + [$.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + + (if onChangedFiles != false then $.testForChangedFiles({ package: onChangedFiles }) else []) + + (if checkVersionBump then [ + $.action('check-version-bump', uses='del-systems/check-if-version-bumped@v1', with={ + token: '${{ github.token }}', + }, ifClause=ifClause), + ] else []) + + (if onChangedFiles != false then std.map(function(step) std.map(function(s) s { 'if': ifClause }, step), buildSteps) else buildSteps) + + $.yarnPublishToRepositories(isPr=true, repositories=repositories, ifClause=ifClause), + permissions={ packages: 'write', contents: 'read', 'pull-requests': 'read' }, + ), + + yarnPublishJob( + image='node:18', + useCredentials=false, + gitCloneRef='${{ github.sha }}', + buildSteps=[$.step('build', 'yarn build')], + repositories=['gynzy'], + onChangedFiles=false, + ifClause=null, + runsOn=null, + ): + local stepIfClause = (if onChangedFiles != false then "steps.changes.outputs.package == 'true'" else null); + $.ghJob( + 'yarn-publish', + image='node:18', + runsOn=runsOn, + useCredentials=false, + steps= + [$.checkoutAndYarn(ref=gitCloneRef, fullClone=false)] + + (if onChangedFiles != false then $.testForChangedFiles({ package: onChangedFiles }) else []) + + (if onChangedFiles != false then std.map(function(step) std.map(function(s) s { 'if': stepIfClause }, step), buildSteps) else buildSteps) + + $.yarnPublishToRepositories(isPr=false, repositories=repositories, ifClause=stepIfClause), + permissions={ packages: 'write', contents: 'read', 'pull-requests': 'read' }, + ifClause=ifClause, + ), } diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml new file mode 100644 index 0000000..174a45d --- /dev/null +++ b/.github/workflows/misc.yml @@ -0,0 +1,28 @@ +"jobs": + "verify-jsonnet-gh-actions": + "container": + "credentials": + "password": "${{ secrets.docker_gcr_io }}" + "username": "_json_key" + "image": "eu.gcr.io/unicorn-985/docker-images_jsonnet:v1" + "runs-on": "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.event.pull_request.head.sha }}" + - "name": "remove-workflows" + "run": "rm -f .github/workflows/*" + - "name": "generate-workflows" + "run": "jsonnet -m .github/workflows/ -S .github.jsonnet;" + - "name": "git workaround" + "run": "git config --global --add safe.directory $PWD" + - "name": "check-jsonnet-diff" + "run": "git diff --exit-code" + - "if": "failure()" + "name": "possible-causes-for-error" + "run": "echo \"Possible causes: \n1. You updated jsonnet files, but did not regenerate the workflows. \nTo fix, run 'yarn github:generate' locally and commit the changes. If this helps, check if your pre-commit hooks work.\n2. You used the wrong jsonnet binary. In this case, the newlines at the end of the files differ.\nTo fix, install the go binary. On mac, run 'brew uninstall jsonnet && brew install jsonnet-go'\"" + "timeout-minutes": 30 +"name": "misc" +"on": +- "pull_request" \ No newline at end of file diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..c20e9f3 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,139 @@ +"jobs": + "test-pr": + "container": + "credentials": + "password": "${{ secrets.docker_gcr_io }}" + "username": "_json_key" + "image": "eu.gcr.io/unicorn-985/docker-images_node14-with-libnss:deploy-5893c6fca68ea35a0a51e855d5a3cb7082ef39fa" + "runs-on": + - "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.event.pull_request.head.sha }}" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "name": "yarn" + "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" + - "name": "setup chrome" + "uses": "browser-actions/setup-chrome@latest" + - "name": "test" + "run": "./node_modules/.bin/ember test" + - "env": + "PR_NUMBER": "${{ github.event.number }}" + "name": "publish" + "run": | + bash -c 'set -xeo pipefail; + + cp package.json package.json.bak; + + VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); + if [[ ! -z "${PR_NUMBER}" ]]; then + echo "Setting tag/version for pr build."; + TAG=pr-$PR_NUMBER; + PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then + echo "Tag version does not match package version. They should match. Exiting"; + exit 1; + fi + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + elif [[ "${GITHUB_REF_TYPE}" == "branch" && "${GITHUB_REF_NAME}" == "main" ]]; then + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + else + exit 1 + fi + + yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; + + mv package.json.bak package.json; + '; + "timeout-minutes": 30 + "yarn-publish-preview": + "container": + "image": "node:18" + "permissions": + "contents": "read" + "packages": "write" + "pull-requests": "read" + "runs-on": "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.event.pull_request.head.sha }}" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "name": "yarn" + "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" + - "name": "check-version-bump" + "uses": "del-systems/check-if-version-bumped@v1" + "with": + "token": "${{ github.token }}" + - "name": "build" + "run": "yarn build" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "env": + "PR_NUMBER": "${{ github.event.number }}" + "name": "publish" + "run": | + bash -c 'set -xeo pipefail; + + cp package.json package.json.bak; + + VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); + if [[ ! -z "${PR_NUMBER}" ]]; then + echo "Setting tag/version for pr build."; + TAG=pr-$PR_NUMBER; + PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then + echo "Tag version does not match package version. They should match. Exiting"; + exit 1; + fi + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + elif [[ "${GITHUB_REF_TYPE}" == "branch" && "${GITHUB_REF_NAME}" == "main" ]]; then + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + else + exit 1 + fi + + yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; + + mv package.json.bak package.json; + '; + "timeout-minutes": 30 +"name": "pr" +"on": +- "pull_request" \ No newline at end of file diff --git a/.github/workflows/publish-prod.yml b/.github/workflows/publish-prod.yml new file mode 100644 index 0000000..95bfec5 --- /dev/null +++ b/.github/workflows/publish-prod.yml @@ -0,0 +1,74 @@ +"jobs": + "yarn-publish": + "container": + "image": "node:18" + "permissions": + "contents": "read" + "packages": "write" + "pull-requests": "read" + "runs-on": "ubuntu-latest" + "steps": + - "name": "Check out repository code" + "uses": "actions/checkout@v3" + "with": + "ref": "${{ github.sha }}" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "name": "yarn" + "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" + - "name": "build" + "run": "yarn build" + - "env": + "NPM_TOKEN": "${{ secrets.npm_token }}" + "name": "set gynzy npm_token" + "run": | + cat < .npmrc + registry=https://npm.gynzy.net/ + always-auth="true" + "//npm.gynzy.net/:_authToken"="${NPM_TOKEN}" + EOF + - "env": {} + "name": "publish" + "run": | + bash -c 'set -xeo pipefail; + + cp package.json package.json.bak; + + VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); + if [[ ! -z "${PR_NUMBER}" ]]; then + echo "Setting tag/version for pr build."; + TAG=pr-$PR_NUMBER; + PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; + elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then + echo "Tag version does not match package version. They should match. Exiting"; + exit 1; + fi + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + elif [[ "${GITHUB_REF_TYPE}" == "branch" && "${GITHUB_REF_NAME}" == "main" ]]; then + echo "Setting tag/version for release/tag build."; + PUBLISHVERSION=$VERSION; + TAG="latest"; + else + exit 1 + fi + + yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; + + mv package.json.bak package.json; + '; + "timeout-minutes": 30 +"name": "publish-prod" +"on": + "push": + "branches": + - "${{ github.event.pull_request.base.ref }}" \ No newline at end of file diff --git a/.github/workflows/publish-tag.yml b/.github/workflows/publish-tag.yml deleted file mode 100644 index 0a5f4da..0000000 --- a/.github/workflows/publish-tag.yml +++ /dev/null @@ -1,29 +0,0 @@ -"jobs": - "publish": - "container": - "credentials": - "password": "${{ secrets.docker_gcr_io }}" - "username": "_json_key" - "image": "eu.gcr.io/unicorn-985/docker-images_node14-with-libnss:deploy-5893c6fca68ea35a0a51e855d5a3cb7082ef39fa" - "runs-on": - - "ubuntu-latest" - "steps": - - "name": "Check out repository code" - "uses": "actions/checkout@v2" - "with": - "ref": "${{ github.sha }}" - - "env": - "NPM_TOKEN": "${{ secrets.npm_token }}" - "name": "set npm_token" - "run": "\ncat < .npmrc\nregistry=https://npm.gynzy.net/\nalways-auth=\"true\"\n\"//npm.gynzy.net/:_authToken\"=\"${NPM_TOKEN}\"\nEOF\n " - - "name": "yarn" - "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" - - "env": {} - "name": "publish" - "run": "\nbash -c 'set -xeo pipefail;\n\nVERSION=$(yarn version --non-interactive 2>/dev/null | grep \"Current version\" | grep -o -P '[0-9a-zA-Z_.-]+$' );\nif [[ ! -z \"${PR_NUMBER}\" ]]; then\n\techo \"Setting tag/version for pr build.\";\n\tTAG=pr-$PR_NUMBER;\n\tPUBLISHVERSION=\"$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER\";\nelif [[ \"${GITHUB_REF_TYPE}\" == \"tag\" ]]; then\n\tif [[ \"${GITHUB_REF_NAME}\" != \"${VERSION}\" ]]; then\n\t echo \"Tag version does not match package version. They should match. Exiting\";\n\t\texit 1;\n\tfi\n\techo \"Setting tag/version for release/tag build.\";\n\tPUBLISHVERSION=$VERSION;\n\tTAG=\"latest\";\nelse\n\texit 1\nfi\n\nyarn publish --non-interactive --no-git-tag-version --tag \"$TAG\" --new-version \"$PUBLISHVERSION\"';\n " - "timeout-minutes": 30 -"name": "publish-tag" -"on": - "push": - "tags": - - "*" \ No newline at end of file diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml deleted file mode 100644 index 30b4300..0000000 --- a/.github/workflows/test-pr.yml +++ /dev/null @@ -1,31 +0,0 @@ -"jobs": - "test-pr": - "container": - "credentials": - "password": "${{ secrets.docker_gcr_io }}" - "username": "_json_key" - "image": "eu.gcr.io/unicorn-985/docker-images_node14-with-libnss:deploy-5893c6fca68ea35a0a51e855d5a3cb7082ef39fa" - "runs-on": - - "ubuntu-latest" - "steps": - - "name": "Check out repository code" - "uses": "actions/checkout@v2" - "with": - "ref": "${{ github.event.pull_request.head.sha }}" - - "env": - "NPM_TOKEN": "${{ secrets.npm_token }}" - "name": "set npm_token" - "run": "\ncat < .npmrc\nregistry=https://npm.gynzy.net/\nalways-auth=\"true\"\n\"//npm.gynzy.net/:_authToken\"=\"${NPM_TOKEN}\"\nEOF\n " - - "name": "yarn" - "run": "yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline || yarn --cache-folder .yarncache --frozen-lockfile --prefer-offline" - - "name": "setup chrome" - "uses": "browser-actions/setup-chrome@latest" - - "name": "test" - "run": "./node_modules/.bin/ember test" - - "env": - "PR_NUMBER": "${{ github.event.number }}" - "name": "publish" - "run": "\nbash -c 'set -xeo pipefail;\n\nVERSION=$(yarn version --non-interactive 2>/dev/null | grep \"Current version\" | grep -o -P '[0-9a-zA-Z_.-]+$' );\nif [[ ! -z \"${PR_NUMBER}\" ]]; then\n\techo \"Setting tag/version for pr build.\";\n\tTAG=pr-$PR_NUMBER;\n\tPUBLISHVERSION=\"$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER\";\nelif [[ \"${GITHUB_REF_TYPE}\" == \"tag\" ]]; then\n\tif [[ \"${GITHUB_REF_NAME}\" != \"${VERSION}\" ]]; then\n\t echo \"Tag version does not match package version. They should match. Exiting\";\n\t\texit 1;\n\tfi\n\techo \"Setting tag/version for release/tag build.\";\n\tPUBLISHVERSION=$VERSION;\n\tTAG=\"latest\";\nelse\n\texit 1\nfi\n\nyarn publish --non-interactive --no-git-tag-version --tag \"$TAG\" --new-version \"$PUBLISHVERSION\"';\n " - "timeout-minutes": 30 -"name": "test-pr" -"on": "pull_request" \ No newline at end of file From ae8449bd76d36de9a130f18d2b37a7cd930bce20 Mon Sep 17 00:00:00 2001 From: Daniel Genis Date: Thu, 19 Oct 2023 12:17:15 +0200 Subject: [PATCH 2/4] chore: bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 160739b..903860a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gynzy/ember-redux", - "version": "6.0.2", + "version": "6.0.3", "description": "ember-cli addon that provides simple redux bindings for ember.js", "scripts": { "build": "ember build", From 7e91c3ba16c750d899626774125e3921376f22af Mon Sep 17 00:00:00 2001 From: Daniel Genis Date: Thu, 19 Oct 2023 12:17:58 +0200 Subject: [PATCH 3/4] docs: add info for release procedure: --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index dcd0d2a..55cff26 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +# Gynzy release procedure + +Step 1: create PR and bump version. +Step 2: test provided PR package in your software +Step 3: merge PR to release the version + # Ember Redux [![Travis][build-badge]][build] [![Code Climate][climate-badge]][climate] [![Score][score-badge]][score] [![Downloads][downloads-badge]][npm] [![npm package][npm-badge]][npm] From 861965855964392e547abd4eb0e5cf9e2430d57d Mon Sep 17 00:00:00 2001 From: Daniel Genis Date: Thu, 19 Oct 2023 12:19:36 +0200 Subject: [PATCH 4/4] chore: remove publish from test-pr already present in another job --- .github.jsonnet | 1 - .github/workflows/pr.yml | 33 --------------------------------- 2 files changed, 34 deletions(-) diff --git a/.github.jsonnet b/.github.jsonnet index 66f56fb..c7b9153 100644 --- a/.github.jsonnet +++ b/.github.jsonnet @@ -9,7 +9,6 @@ local testJob = util.ghJob( util.checkoutAndYarn(ref='${{ github.event.pull_request.head.sha }}', fullClone=false), util.action('setup chrome', 'browser-actions/setup-chrome@latest'), util.step('test', './node_modules/.bin/ember test'), - util.yarnPublish(isPr=true), ], runsOn=['ubuntu-latest'], // it's public fork. don't use private runners for public fork ); diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c20e9f3..de5713b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,39 +27,6 @@ "uses": "browser-actions/setup-chrome@latest" - "name": "test" "run": "./node_modules/.bin/ember test" - - "env": - "PR_NUMBER": "${{ github.event.number }}" - "name": "publish" - "run": | - bash -c 'set -xeo pipefail; - - cp package.json package.json.bak; - - VERSION=$(yarn version --non-interactive 2>/dev/null | grep "Current version" | grep -o -P '[0-9a-zA-Z_.-]+$' ); - if [[ ! -z "${PR_NUMBER}" ]]; then - echo "Setting tag/version for pr build."; - TAG=pr-$PR_NUMBER; - PUBLISHVERSION="$VERSION-pr$PR_NUMBER.$GITHUB_RUN_NUMBER"; - elif [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then - if [[ "${GITHUB_REF_NAME}" != "${VERSION}" ]]; then - echo "Tag version does not match package version. They should match. Exiting"; - exit 1; - fi - echo "Setting tag/version for release/tag build."; - PUBLISHVERSION=$VERSION; - TAG="latest"; - elif [[ "${GITHUB_REF_TYPE}" == "branch" && "${GITHUB_REF_NAME}" == "main" ]]; then - echo "Setting tag/version for release/tag build."; - PUBLISHVERSION=$VERSION; - TAG="latest"; - else - exit 1 - fi - - yarn publish --non-interactive --no-git-tag-version --tag "$TAG" --new-version "$PUBLISHVERSION"; - - mv package.json.bak package.json; - '; "timeout-minutes": 30 "yarn-publish-preview": "container":