Skip to content
Build and test for release

Build and test for release #404

Sign in to view logs

Build and test for release

Build and test for release #404

Workflow file for this run

.github/workflows/build-and-test_release.yml at 1a2484a
#
# THIS FILE IS GENERATED, PLEASE DO NOT EDIT.
#
# Copyright 2022 Flant JSC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# on push to default branch or on tags
name: Build and test for release
on:
workflow_dispatch:
inputs:
issue_id:
description: 'Id of issue where label was set'
required: false
issue_number:
description: 'Number of issue where label was set'
required: false
comment_id:
description: 'Id of comment in issue where to put workflow run status'
required: false
env:
# <template: werf_envs>
WERF_CHANNEL: "ea"
WERF_ENV: "FE"
TEST_TIMEOUT: "15m"
# Use fixed string 'sys/deckhouse-oss' for repo name. ${CI_PROJECT_PATH} is not available here in GitHub.
DEV_REGISTRY_PATH: "${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/sys/deckhouse-oss"
# Registry for additional repositories used for testing Github Actions workflows.
GHA_TEST_REGISTRY_PATH: "ghcr.io/${{ github.repository }}"
# </template: werf_envs>
# <template: git_source_envs>
# source repo contains should creds for repo for ex https://user:password@my-repo.com/group
SOURCE_REPO: "${{secrets.SOURCE_REPO}}"
# </template: git_source_envs>
# Cancel in-progress jobs for the same tag/branch.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
started_at:
name: Save start timestamp
outputs:
started_at: ${{ steps.started_at.outputs.started_at }}
runs-on: "ubuntu-latest"
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: git_info_job>
git_info:
name: Get git info
runs-on: ubuntu-latest
outputs:
ci_commit_tag: ${{ steps.git_info.outputs.ci_commit_tag }}
ci_commit_branch: ${{ steps.git_info.outputs.ci_commit_branch }}
ci_commit_ref_name: ${{ steps.git_info.outputs.ci_commit_ref_name }}
ci_commit_ref_slug: ${{ steps.git_info.outputs.ci_commit_ref_slug }}
ref_full: ${{ steps.git_info.outputs.ref_full }}
github_sha: ${{ steps.git_info.outputs.github_sha }}
pr_number: ${{ steps.git_info.outputs.pr_number }}
# Skip the CI for automation PRs, e.g. changelog
if: ${{ github.event.pull_request.user.login != 'deckhouse-BOaTswain' }}
steps:
- id: git_info
name: Get tag name and SHA
uses: actions/github-script@v6.4.1
with:
script: |
const { GITHUB_REF_TYPE, GITHUB_REF_NAME, GITHUB_REF } = process.env
let refSlug = ''
let refName = ''
let refFull = ''
let githubBranch = ''
let githubTag = ''
let githubSHA = ''
let prNumber = ''
if (context.eventName === "workflow_dispatch" && context.payload.inputs && context.payload.inputs.pull_request_ref) {
// Trigger: workflow_dispatch with pull_request_ref.
// Extract pull request number from 'refs/pull/<NUM>/merge'
prNumber = context.payload.inputs.pull_request_ref.replace('refs/pull/', '').replace('/merge', '').replace('/head', '')
refSlug = `pr${prNumber}`
refName = context.payload.inputs.ci_commit_ref_name
refFull = context.payload.inputs.pull_request_ref
githubBranch = refName
githubSHA = context.payload.inputs.pull_request_sha
core.info(`workflow_dispatch event: set git info from inputs. inputs: ${JSON.stringify(context.payload.inputs)}`)
} else if (context.eventName === "pull_request" || context.eventName === "pull_request_target" ) {
// For PRs from forks, tag images with `prXXX` to avoid clashes between branches.
const targetRepo = context.payload.repository.full_name;
const prRepo = context.payload.pull_request.head.repo.full_name
const prRef = context.payload.pull_request.head.ref
refSlug = `pr${context.issue.number}`;
refName = (prRepo === targetRepo) ? prRef : refSlug;
refFull = `refs/pull/${context.issue.number}/head`
githubBranch = refName
githubSHA = context.payload.pull_request.head.sha
core.info(`pull request event: set git info from pull_request.head. pr:${prRepo}:${prRef} target:${targetRepo}:${context.ref}`)
prNumber = context.issue.number
} else {
// Other triggers: workflow_dispatch without pull_request_ref, schedule, push...
// refName is 'main' or tag name, so slugification is not necessary.
refSlug = GITHUB_REF_NAME
refName = GITHUB_REF_NAME
refFull = GITHUB_REF
githubTag = GITHUB_REF_TYPE == "tag" ? refName : ""
githubBranch = GITHUB_REF_TYPE == "branch" ? refName : ""
githubSHA = context.sha
core.info(`${context.eventName} event: set git info from context: ${JSON.stringify({GITHUB_REF_NAME, GITHUB_REF_TYPE, sha: context.sha })}`)
}
core.setCommandEcho(true)
core.setOutput('ci_commit_ref_slug', refSlug)
core.setOutput('ci_commit_ref_name', refName)
core.setOutput(`ci_commit_tag`, githubTag)
core.setOutput(`ci_commit_branch`, githubBranch)
core.setOutput(`ref_full`, refFull)
core.setOutput('github_sha', githubSHA)
core.setOutput('pr_number', prNumber)
core.setCommandEcho(false)
# </template: git_info_job>
comment_on_start:
name: Update issue comment
runs-on: ubuntu-latest
steps:
# <template: checkout_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
# </template: checkout_step>
# <template: update_comment_on_start>
- name: Update comment on start
if: ${{ github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const name = 'Build and test for release';
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnStart({github, context, core, name})
# </template: update_comment_on_start>
go_generate:
name: Go Generate
needs:
- git_info
# <template: go_generate_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
# </template: checkout_step>
# <template: login_readonly_registry_step>
- name: Check readonly registry credentials
id: check_readonly_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_READ_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_READ_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to readonly registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_readonly_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_READ_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_READ_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_READ_PASSWORD }}
logout: false
# </template: login_readonly_registry_step>
# <template: werf_install_step>
- name: Install werf CLI
uses: werf/actions/install@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
# </template: werf_install_step>
- name: Set up Go 1.20
uses: actions/setup-go@v3
with:
go-version: '1.20'
- name: Run go generate
run: |
(cd tools && go generate)
(cd modules/500-upmeter/hooks/smokemini/internal/snapshot && go generate)
- name: Check generated code
run: |
git diff --exit-code
# </template: go_generate_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Go Generate';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
workflow_render:
name: Render workflow
needs:
- git_info
# <template: workflow_render_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
# </template: checkout_step>
# <template: login_readonly_registry_step>
- name: Check readonly registry credentials
id: check_readonly_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_READ_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_READ_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to readonly registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_readonly_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_READ_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_READ_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_READ_PASSWORD }}
logout: false
# </template: login_readonly_registry_step>
- name: Render GitHub workflow
run: |
cd .github
./render-workflows.sh
- name: Check rendered files
run: |
git diff --exit-code
# </template: workflow_render_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Render workflow';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
build_fe:
name: Build FE
needs:
- git_info
- go_generate
- workflow_render
env:
WERF_ENV: "FE"
# <template: build_template>
runs-on: [self-hosted, regular]
outputs:
tests_image_name: ${{ steps.build.outputs.tests_image_name }}
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_readonly_registry_step>
- name: Check readonly registry credentials
id: check_readonly_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_READ_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_READ_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to readonly registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_readonly_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_READ_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_READ_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_READ_PASSWORD }}
logout: false
# </template: login_readonly_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
# <template: werf_install_step>
- name: Install werf CLI
uses: werf/actions/install@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
# </template: werf_install_step>
- name: Build and push deckhouse images
id: build
env:
DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}}
CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}}
CI_COMMIT_REF_NAME: ${{needs.git_info.outputs.ci_commit_ref_name}}
CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}}
run: |
# Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'.
REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-}
if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then
# REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'.
REPO_SUFFIX=
fi
# Put tags on produced images and push to dev and release repositories.
#
# There are 2 modes: "dev" and "release".
# The "dev" mode builds branches only:
# - Build using dev-registry as primary and deckhouse registry as secondary.
# - Push dev and dev/install images with prNUM tags and push to dev-registry.
# The "release" mode builds branches and tags:
# - Build using dev-registry as primary and deckhouse registry as secondary.
# - Push dev and dev/install images to dev-registry with tag equal to a branch name (main or release-X.Y).
# - Build using deckhouse registry as primary and dev-registry as secondary.
# - Push dev, dev/install and release-channel-version images to deckhouse registry with tag equels to a Git tag.
# SRC_NAME is a name of image from werf.yaml.
# SRC is a source image name (stage name from werf build report).
# DST is an image name for docker push.
function pull_push_rmi() {
SRC_NAME=$1
SRC=$2
DST=$3
echo "⚓️ 📥 [$(date -u)] Pull '${SRC_NAME}' image as ${SRC}."
docker pull ${SRC}
echo "⚓️ 🏷 [$(date -u)] Tag '${SRC_NAME}' image as ${DST}."
docker image tag ${SRC} ${DST}
echo "⚓️ 📤 [$(date -u)] Push '${SRC_NAME}' image as ${DST}."
docker image push ${DST}
echo "⚓️ 🧹 [$(date -u)] Remove local tag for '${SRC_NAME}'."
docker image rmi ${DST} || true;
}
if [[ -n "${DEV_REGISTRY_PATH}" ]]; then export WERF_REPO="${DEV_REGISTRY_PATH}"; fi
type werf && source $(werf ci-env github --verbose --as-file)
# CE/EE/FE -> ce/ee/fe
REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]')
# Registry path to publish images for Git branches.
BRANCH_REGISTRY_PATH=
# Registry path to publish images for Git tags.
SEMVER_REGISTRY_PATH=
if [[ -n ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then
# Build using dev-registry as primary repo and prod registry as secondary (ro) repo.
# This build will put stages to "dev" registry. If "dev" registry is empty, existing stages are copied from prod registry.
SECONDARY_REPO="--secondary-repo ${DECKHOUSE_REGISTRY_HOST}/deckhouse/${REGISTRY_SUFFIX}"
if [[ -n "${CI_COMMIT_BRANCH}" && ! "${CI_COMMIT_BRANCH}" =~ ^(main|release-.+)$ ]]; then
SECONDARY_REPO=
fi
werf build \
${SECONDARY_REPO} \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}"
SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse"
else
# DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo.
# Build using dev-registry as a single primary repo and push:
# - branches to Dev registry to run e2e tests.
# - semver tags to Github Container Registry for testing release process.
werf build \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}"
SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}"
echo "⚓️ 🧪 [$(date -u)] DECKHOUSE_REGISTRY_HOST is empty. Publish to Github Container Registry '${PROD_REGISTRY_PATH}'"
fi
# Publish images for Git branch.
if [[ -n "${CI_COMMIT_BRANCH}" ]]; then
# CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch.
# Use it as image tag. Add suffix to not overlap with PRs in main repo.
IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}}
echo "⚓️ 💫 [$(date -u)] Publish images to dev-registry for branch '${CI_COMMIT_BRANCH}' and edition '${WERF_ENV}' using tag '${IMAGE_TAG}' ..."
echo "⚓️ 💫 [$(date -u)] Publish 'dev' image to dev-registry using tag ${IMAGE_TAG}".
DECKHOUSE_IMAGE_SRC="$(jq -r '.Images."dev".DockerImageName' images_tags_werf.json)"
DECKHOUSE_IMAGE=${BRANCH_REGISTRY_PATH}:${IMAGE_TAG}
pull_push_rmi 'dev' ${DECKHOUSE_IMAGE_SRC} ${DECKHOUSE_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'dev/install' image to dev-registry using tag ${IMAGE_TAG}".
INSTALL_IMAGE_SRC="$(jq -r '.Images."dev/install".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG}
pull_push_rmi 'dev/install' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'e2e-terraform' image to dev-registry using tag ${IMAGE_TAG}".
INSTALL_IMAGE_SRC="$(jq -r '.Images."e2e-terraform".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${BRANCH_REGISTRY_PATH}/e2e-terraform:${IMAGE_TAG}
pull_push_rmi 'e2e-terraform' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
fi
# Publish images for Git tag.
if [[ -n "${CI_COMMIT_TAG}" ]]; then
# The Git tag may contain a '+' sign, so use slugify for this situation.
# Slugify doesn't change a tag with safe-only characters.
IMAGE_TAG=$(werf slugify --format docker-tag "${CI_COMMIT_TAG}")
echo "⚓️ 💫 [$(date -u)] Publish images for Git tag '${CI_COMMIT_TAG}' and registry suffix '${REGISTRY_SUFFIX}' using tag '${IMAGE_TAG}' ..."
if [[ -n ${DECKHOUSE_REGISTRY_HOST} ]] ; then
# Copy stages to prod registry from dev registry.
werf build \
--repo ${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX} \
--secondary-repo $WERF_REPO \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
fi
# Note: do not run second werf build for test repo, as it has no secondary repo.
echo "⚓️ 💫 [$(date -u)] Publish 'dev' image to deckhouse registry using tag ${IMAGE_TAG} ..."
DECKHOUSE_IMAGE_SRC="$(jq -r '.Images."dev".DockerImageName' images_tags_werf.json)"
DECKHOUSE_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}:${IMAGE_TAG}
pull_push_rmi 'dev' ${DECKHOUSE_IMAGE_SRC} ${DECKHOUSE_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'dev/install' image to deckhouse registry using tag ${IMAGE_TAG} ..."
INSTALL_IMAGE_SRC="$(jq -r '.Images."dev/install".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${IMAGE_TAG}
pull_push_rmi 'dev/install' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'release-channel-version' image to deckhouse registry using tag ${IMAGE_TAG} ..."
RELEASE_CHANNEL_IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)"
RELEASE_CHANNEL_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/release-channel:${IMAGE_TAG}
pull_push_rmi 'release-channel-version' ${RELEASE_CHANNEL_IMAGE_SRC} ${RELEASE_CHANNEL_IMAGE}
fi
# Save 'tests' image name to pass it as output for 'tests' jobs.
TESTS_IMAGE_NAME="$(jq -r '.Images."tests".DockerImageName' images_tags_werf.json)"
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
# Encode as gzip+base64 to evade github's SecretMasker error: "Skip output since it may contain secret".
echo "tests_image_name=$(echo ${TESTS_IMAGE_NAME} | gzip | base64 -w0)" >> $GITHUB_OUTPUT
- name: Cleanup
if: ${{ always() }}
run: |
rm -f images_tags_werf.json
# </template: build_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Build FE';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
build_ee:
name: Build EE
needs:
- git_info
- go_generate
- workflow_render
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
env:
WERF_ENV: "EE"
# <template: build_template>
runs-on: [self-hosted, regular]
outputs:
tests_image_name: ${{ steps.build.outputs.tests_image_name }}
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_readonly_registry_step>
- name: Check readonly registry credentials
id: check_readonly_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_READ_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_READ_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to readonly registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_readonly_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_READ_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_READ_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_READ_PASSWORD }}
logout: false
# </template: login_readonly_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
# <template: werf_install_step>
- name: Install werf CLI
uses: werf/actions/install@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
# </template: werf_install_step>
- name: Build and push deckhouse images
id: build
env:
DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}}
CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}}
CI_COMMIT_REF_NAME: ${{needs.git_info.outputs.ci_commit_ref_name}}
CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}}
run: |
# Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'.
REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-}
if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then
# REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'.
REPO_SUFFIX=
fi
# Put tags on produced images and push to dev and release repositories.
#
# There are 2 modes: "dev" and "release".
# The "dev" mode builds branches only:
# - Build using dev-registry as primary and deckhouse registry as secondary.
# - Push dev and dev/install images with prNUM tags and push to dev-registry.
# The "release" mode builds branches and tags:
# - Build using dev-registry as primary and deckhouse registry as secondary.
# - Push dev and dev/install images to dev-registry with tag equal to a branch name (main or release-X.Y).
# - Build using deckhouse registry as primary and dev-registry as secondary.
# - Push dev, dev/install and release-channel-version images to deckhouse registry with tag equels to a Git tag.
# SRC_NAME is a name of image from werf.yaml.
# SRC is a source image name (stage name from werf build report).
# DST is an image name for docker push.
function pull_push_rmi() {
SRC_NAME=$1
SRC=$2
DST=$3
echo "⚓️ 📥 [$(date -u)] Pull '${SRC_NAME}' image as ${SRC}."
docker pull ${SRC}
echo "⚓️ 🏷 [$(date -u)] Tag '${SRC_NAME}' image as ${DST}."
docker image tag ${SRC} ${DST}
echo "⚓️ 📤 [$(date -u)] Push '${SRC_NAME}' image as ${DST}."
docker image push ${DST}
echo "⚓️ 🧹 [$(date -u)] Remove local tag for '${SRC_NAME}'."
docker image rmi ${DST} || true;
}
if [[ -n "${DEV_REGISTRY_PATH}" ]]; then export WERF_REPO="${DEV_REGISTRY_PATH}"; fi
type werf && source $(werf ci-env github --verbose --as-file)
# CE/EE/FE -> ce/ee/fe
REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]')
# Registry path to publish images for Git branches.
BRANCH_REGISTRY_PATH=
# Registry path to publish images for Git tags.
SEMVER_REGISTRY_PATH=
if [[ -n ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then
# Build using dev-registry as primary repo and prod registry as secondary (ro) repo.
# This build will put stages to "dev" registry. If "dev" registry is empty, existing stages are copied from prod registry.
SECONDARY_REPO="--secondary-repo ${DECKHOUSE_REGISTRY_HOST}/deckhouse/${REGISTRY_SUFFIX}"
if [[ -n "${CI_COMMIT_BRANCH}" && ! "${CI_COMMIT_BRANCH}" =~ ^(main|release-.+)$ ]]; then
SECONDARY_REPO=
fi
werf build \
${SECONDARY_REPO} \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}"
SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse"
else
# DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo.
# Build using dev-registry as a single primary repo and push:
# - branches to Dev registry to run e2e tests.
# - semver tags to Github Container Registry for testing release process.
werf build \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}"
SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}"
echo "⚓️ 🧪 [$(date -u)] DECKHOUSE_REGISTRY_HOST is empty. Publish to Github Container Registry '${PROD_REGISTRY_PATH}'"
fi
# Publish images for Git branch.
if [[ -n "${CI_COMMIT_BRANCH}" ]]; then
# CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch.
# Use it as image tag. Add suffix to not overlap with PRs in main repo.
IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}}
echo "⚓️ 💫 [$(date -u)] Publish images to dev-registry for branch '${CI_COMMIT_BRANCH}' and edition '${WERF_ENV}' using tag '${IMAGE_TAG}' ..."
echo "⚓️ 💫 [$(date -u)] Publish 'dev' image to dev-registry using tag ${IMAGE_TAG}".
DECKHOUSE_IMAGE_SRC="$(jq -r '.Images."dev".DockerImageName' images_tags_werf.json)"
DECKHOUSE_IMAGE=${BRANCH_REGISTRY_PATH}:${IMAGE_TAG}
pull_push_rmi 'dev' ${DECKHOUSE_IMAGE_SRC} ${DECKHOUSE_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'dev/install' image to dev-registry using tag ${IMAGE_TAG}".
INSTALL_IMAGE_SRC="$(jq -r '.Images."dev/install".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG}
pull_push_rmi 'dev/install' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'e2e-terraform' image to dev-registry using tag ${IMAGE_TAG}".
INSTALL_IMAGE_SRC="$(jq -r '.Images."e2e-terraform".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${BRANCH_REGISTRY_PATH}/e2e-terraform:${IMAGE_TAG}
pull_push_rmi 'e2e-terraform' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
fi
# Publish images for Git tag.
if [[ -n "${CI_COMMIT_TAG}" ]]; then
# The Git tag may contain a '+' sign, so use slugify for this situation.
# Slugify doesn't change a tag with safe-only characters.
IMAGE_TAG=$(werf slugify --format docker-tag "${CI_COMMIT_TAG}")
echo "⚓️ 💫 [$(date -u)] Publish images for Git tag '${CI_COMMIT_TAG}' and registry suffix '${REGISTRY_SUFFIX}' using tag '${IMAGE_TAG}' ..."
if [[ -n ${DECKHOUSE_REGISTRY_HOST} ]] ; then
# Copy stages to prod registry from dev registry.
werf build \
--repo ${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX} \
--secondary-repo $WERF_REPO \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
fi
# Note: do not run second werf build for test repo, as it has no secondary repo.
echo "⚓️ 💫 [$(date -u)] Publish 'dev' image to deckhouse registry using tag ${IMAGE_TAG} ..."
DECKHOUSE_IMAGE_SRC="$(jq -r '.Images."dev".DockerImageName' images_tags_werf.json)"
DECKHOUSE_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}:${IMAGE_TAG}
pull_push_rmi 'dev' ${DECKHOUSE_IMAGE_SRC} ${DECKHOUSE_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'dev/install' image to deckhouse registry using tag ${IMAGE_TAG} ..."
INSTALL_IMAGE_SRC="$(jq -r '.Images."dev/install".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${IMAGE_TAG}
pull_push_rmi 'dev/install' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'release-channel-version' image to deckhouse registry using tag ${IMAGE_TAG} ..."
RELEASE_CHANNEL_IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)"
RELEASE_CHANNEL_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/release-channel:${IMAGE_TAG}
pull_push_rmi 'release-channel-version' ${RELEASE_CHANNEL_IMAGE_SRC} ${RELEASE_CHANNEL_IMAGE}
fi
# Save 'tests' image name to pass it as output for 'tests' jobs.
TESTS_IMAGE_NAME="$(jq -r '.Images."tests".DockerImageName' images_tags_werf.json)"
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
# Encode as gzip+base64 to evade github's SecretMasker error: "Skip output since it may contain secret".
echo "tests_image_name=$(echo ${TESTS_IMAGE_NAME} | gzip | base64 -w0)" >> $GITHUB_OUTPUT
- name: Cleanup
if: ${{ always() }}
run: |
rm -f images_tags_werf.json
# </template: build_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Build EE';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
build_ce:
name: Build CE
needs:
- git_info
- go_generate
- workflow_render
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
env:
WERF_ENV: "CE"
# <template: build_template>
runs-on: [self-hosted, regular]
outputs:
tests_image_name: ${{ steps.build.outputs.tests_image_name }}
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_readonly_registry_step>
- name: Check readonly registry credentials
id: check_readonly_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_READ_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_READ_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to readonly registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_readonly_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_READ_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_READ_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_READ_PASSWORD }}
logout: false
# </template: login_readonly_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
# <template: werf_install_step>
- name: Install werf CLI
uses: werf/actions/install@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
# </template: werf_install_step>
- name: Build and push deckhouse images
id: build
env:
DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}}
CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}}
CI_COMMIT_REF_NAME: ${{needs.git_info.outputs.ci_commit_ref_name}}
CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}}
run: |
# Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'.
REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-}
if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then
# REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'.
REPO_SUFFIX=
fi
# Put tags on produced images and push to dev and release repositories.
#
# There are 2 modes: "dev" and "release".
# The "dev" mode builds branches only:
# - Build using dev-registry as primary and deckhouse registry as secondary.
# - Push dev and dev/install images with prNUM tags and push to dev-registry.
# The "release" mode builds branches and tags:
# - Build using dev-registry as primary and deckhouse registry as secondary.
# - Push dev and dev/install images to dev-registry with tag equal to a branch name (main or release-X.Y).
# - Build using deckhouse registry as primary and dev-registry as secondary.
# - Push dev, dev/install and release-channel-version images to deckhouse registry with tag equels to a Git tag.
# SRC_NAME is a name of image from werf.yaml.
# SRC is a source image name (stage name from werf build report).
# DST is an image name for docker push.
function pull_push_rmi() {
SRC_NAME=$1
SRC=$2
DST=$3
echo "⚓️ 📥 [$(date -u)] Pull '${SRC_NAME}' image as ${SRC}."
docker pull ${SRC}
echo "⚓️ 🏷 [$(date -u)] Tag '${SRC_NAME}' image as ${DST}."
docker image tag ${SRC} ${DST}
echo "⚓️ 📤 [$(date -u)] Push '${SRC_NAME}' image as ${DST}."
docker image push ${DST}
echo "⚓️ 🧹 [$(date -u)] Remove local tag for '${SRC_NAME}'."
docker image rmi ${DST} || true;
}
if [[ -n "${DEV_REGISTRY_PATH}" ]]; then export WERF_REPO="${DEV_REGISTRY_PATH}"; fi
type werf && source $(werf ci-env github --verbose --as-file)
# CE/EE/FE -> ce/ee/fe
REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]')
# Registry path to publish images for Git branches.
BRANCH_REGISTRY_PATH=
# Registry path to publish images for Git tags.
SEMVER_REGISTRY_PATH=
if [[ -n ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then
# Build using dev-registry as primary repo and prod registry as secondary (ro) repo.
# This build will put stages to "dev" registry. If "dev" registry is empty, existing stages are copied from prod registry.
SECONDARY_REPO="--secondary-repo ${DECKHOUSE_REGISTRY_HOST}/deckhouse/${REGISTRY_SUFFIX}"
if [[ -n "${CI_COMMIT_BRANCH}" && ! "${CI_COMMIT_BRANCH}" =~ ^(main|release-.+)$ ]]; then
SECONDARY_REPO=
fi
werf build \
${SECONDARY_REPO} \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}"
SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse"
else
# DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo.
# Build using dev-registry as a single primary repo and push:
# - branches to Dev registry to run e2e tests.
# - semver tags to Github Container Registry for testing release process.
werf build \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}"
SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}"
echo "⚓️ 🧪 [$(date -u)] DECKHOUSE_REGISTRY_HOST is empty. Publish to Github Container Registry '${PROD_REGISTRY_PATH}'"
fi
# Publish images for Git branch.
if [[ -n "${CI_COMMIT_BRANCH}" ]]; then
# CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch.
# Use it as image tag. Add suffix to not overlap with PRs in main repo.
IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}}
echo "⚓️ 💫 [$(date -u)] Publish images to dev-registry for branch '${CI_COMMIT_BRANCH}' and edition '${WERF_ENV}' using tag '${IMAGE_TAG}' ..."
echo "⚓️ 💫 [$(date -u)] Publish 'dev' image to dev-registry using tag ${IMAGE_TAG}".
DECKHOUSE_IMAGE_SRC="$(jq -r '.Images."dev".DockerImageName' images_tags_werf.json)"
DECKHOUSE_IMAGE=${BRANCH_REGISTRY_PATH}:${IMAGE_TAG}
pull_push_rmi 'dev' ${DECKHOUSE_IMAGE_SRC} ${DECKHOUSE_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'dev/install' image to dev-registry using tag ${IMAGE_TAG}".
INSTALL_IMAGE_SRC="$(jq -r '.Images."dev/install".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG}
pull_push_rmi 'dev/install' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'e2e-terraform' image to dev-registry using tag ${IMAGE_TAG}".
INSTALL_IMAGE_SRC="$(jq -r '.Images."e2e-terraform".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${BRANCH_REGISTRY_PATH}/e2e-terraform:${IMAGE_TAG}
pull_push_rmi 'e2e-terraform' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
fi
# Publish images for Git tag.
if [[ -n "${CI_COMMIT_TAG}" ]]; then
# The Git tag may contain a '+' sign, so use slugify for this situation.
# Slugify doesn't change a tag with safe-only characters.
IMAGE_TAG=$(werf slugify --format docker-tag "${CI_COMMIT_TAG}")
echo "⚓️ 💫 [$(date -u)] Publish images for Git tag '${CI_COMMIT_TAG}' and registry suffix '${REGISTRY_SUFFIX}' using tag '${IMAGE_TAG}' ..."
if [[ -n ${DECKHOUSE_REGISTRY_HOST} ]] ; then
# Copy stages to prod registry from dev registry.
werf build \
--repo ${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX} \
--secondary-repo $WERF_REPO \
--parallel=true --parallel-tasks-limit=5 \
--report-path images_tags_werf.json
fi
# Note: do not run second werf build for test repo, as it has no secondary repo.
echo "⚓️ 💫 [$(date -u)] Publish 'dev' image to deckhouse registry using tag ${IMAGE_TAG} ..."
DECKHOUSE_IMAGE_SRC="$(jq -r '.Images."dev".DockerImageName' images_tags_werf.json)"
DECKHOUSE_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}:${IMAGE_TAG}
pull_push_rmi 'dev' ${DECKHOUSE_IMAGE_SRC} ${DECKHOUSE_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'dev/install' image to deckhouse registry using tag ${IMAGE_TAG} ..."
INSTALL_IMAGE_SRC="$(jq -r '.Images."dev/install".DockerImageName' images_tags_werf.json)"
INSTALL_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${IMAGE_TAG}
pull_push_rmi 'dev/install' ${INSTALL_IMAGE_SRC} ${INSTALL_IMAGE}
echo "⚓️ 💫 [$(date -u)] Publish 'release-channel-version' image to deckhouse registry using tag ${IMAGE_TAG} ..."
RELEASE_CHANNEL_IMAGE_SRC="$(jq -r '.Images."release-channel-version".DockerImageName' images_tags_werf.json)"
RELEASE_CHANNEL_IMAGE=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/release-channel:${IMAGE_TAG}
pull_push_rmi 'release-channel-version' ${RELEASE_CHANNEL_IMAGE_SRC} ${RELEASE_CHANNEL_IMAGE}
fi
# Save 'tests' image name to pass it as output for 'tests' jobs.
TESTS_IMAGE_NAME="$(jq -r '.Images."tests".DockerImageName' images_tags_werf.json)"
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
# Encode as gzip+base64 to evade github's SecretMasker error: "Skip output since it may contain secret".
echo "tests_image_name=$(echo ${TESTS_IMAGE_NAME} | gzip | base64 -w0)" >> $GITHUB_OUTPUT
- name: Cleanup
if: ${{ always() }}
run: |
rm -f images_tags_werf.json
# </template: build_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Build CE';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
doc_web_build:
name: Doc web build
# Wait for success build of modules.
needs:
- git_info
# <template: web_build_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run doc web build
uses: werf/actions/build@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
env:
WERF_DIR: "docs/documentation"
WERF_LOG_VERBOSE: "on"
WERF_REPO: "${{ steps.check_rw_registry.outputs.web_registry_path }}"
WERF_SECONDARY_REPO: "${{ steps.check_dev_registry.outputs.web_registry_path }}"
# </template: web_build_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Doc web build';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
main_web_build:
name: Main web build
# Wait for success build of modules.
needs:
- git_info
# <template: web_build_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run main web build
uses: werf/actions/build@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
env:
WERF_DIR: "docs/site"
WERF_LOG_VERBOSE: "on"
WERF_REPO: "${{ steps.check_rw_registry.outputs.web_registry_path }}"
WERF_SECONDARY_REPO: "${{ steps.check_dev_registry.outputs.web_registry_path }}"
# </template: web_build_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Main web build';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
tests:
name: Tests
needs:
- git_info
# <template: tests_before_build_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run tests
env:
TESTS_IMAGE_NAME: "registry.deckhouse.io/base_images/golang:1.18.5-buster@sha256:eb4b3bd59da4604b0ab27d3837e39dacfba4a48cdd289419a3c49993c0eadba4"
run: |
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
echo "⚓️ 📥 [$(date -u)] Pull 'tests' image..."
docker pull ${TESTS_IMAGE_NAME}
echo "⚓️ 🏎 [$(date -u)] Run tests..."
docker run -w /deckhouse -v ${{github.workspace}}:/deckhouse -e "TERM=xterm-256color" -v ~/go-pkg-cache:/go/pkg ${TESTS_IMAGE_NAME} make tests-modules
# </template: tests_before_build_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Tests';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
matrix_tests:
name: Matrix tests
needs:
- git_info
# <template: tests_before_build_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run tests
env:
TESTS_IMAGE_NAME: "registry.deckhouse.io/base_images/golang:1.18.5-buster@sha256:eb4b3bd59da4604b0ab27d3837e39dacfba4a48cdd289419a3c49993c0eadba4"
run: |
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
echo "⚓️ 📥 [$(date -u)] Pull 'tests' image..."
docker pull ${TESTS_IMAGE_NAME}
echo "⚓️ 🏎 [$(date -u)] Run tests..."
docker run -w /deckhouse -v ${{github.workspace}}:/deckhouse -e "TERM=xterm-256color" -v ~/go-pkg-cache:/go/pkg -v ~/deckhouse-bin-cache:/deckhouse/bin ${TESTS_IMAGE_NAME} make tests-matrix
# </template: tests_before_build_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Matrix tests';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
dhctl_tests:
name: Dhctl Tests
needs:
- git_info
- build_fe
# <template: tests_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run tests
env:
TESTS_IMAGE_NAME: ${{needs.build_fe.outputs.tests_image_name}}
run: |
if [[ -z ${TESTS_IMAGE_NAME} ]] ; then
echo "TESTS_IMAGE_NAME is empty"
exit 1
fi
# Decode image name from gzip+base64.
TESTS_IMAGE_NAME=$(echo ${TESTS_IMAGE_NAME} | base64 -d | gunzip)
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
echo "⚓️ 📥 [$(date -u)] Pull 'tests' image..."
docker pull ${TESTS_IMAGE_NAME}
echo "⚓️ 🏎 [$(date -u)] Run tests..."
docker run -w /deckhouse/dhctl -v ~/go-pkg-cache:/go/pkg ${TESTS_IMAGE_NAME} make ci
# </template: tests_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Dhctl Tests';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
golangci_lint:
name: GolangCI Lint
needs:
- git_info
- build_fe
# <template: tests_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run tests
env:
TESTS_IMAGE_NAME: ${{needs.build_fe.outputs.tests_image_name}}
run: |
if [[ -z ${TESTS_IMAGE_NAME} ]] ; then
echo "TESTS_IMAGE_NAME is empty"
exit 1
fi
# Decode image name from gzip+base64.
TESTS_IMAGE_NAME=$(echo ${TESTS_IMAGE_NAME} | base64 -d | gunzip)
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
echo "⚓️ 📥 [$(date -u)] Pull 'tests' image..."
docker pull ${TESTS_IMAGE_NAME}
echo "⚓️ 🏎 [$(date -u)] Run tests..."
docker run -w /deckhouse -v ${{github.workspace}}:/deckhouse -e "TERM=xterm-256color" -v ~/go-pkg-cache:/go/pkg ${TESTS_IMAGE_NAME} sh -c "go generate tools/register.go && golangci-lint run"
# </template: tests_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'GolangCI Lint';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
openapi_test_cases:
name: OpenAPI Test Cases
needs:
- git_info
- build_fe
# <template: tests_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run tests
env:
TESTS_IMAGE_NAME: ${{needs.build_fe.outputs.tests_image_name}}
run: |
if [[ -z ${TESTS_IMAGE_NAME} ]] ; then
echo "TESTS_IMAGE_NAME is empty"
exit 1
fi
# Decode image name from gzip+base64.
TESTS_IMAGE_NAME=$(echo ${TESTS_IMAGE_NAME} | base64 -d | gunzip)
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
echo "⚓️ 📥 [$(date -u)] Pull 'tests' image..."
docker pull ${TESTS_IMAGE_NAME}
echo "⚓️ 🏎 [$(date -u)] Run tests..."
docker run -v ${{github.workspace}}:/deckhouse -w /deckhouse -v ~/go-pkg-cache:/go/pkg ${TESTS_IMAGE_NAME} ginkgo -vet=off ./testing/openapi_cases/
# </template: tests_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'OpenAPI Test Cases';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
web_links_test:
name: Web links test
if: ${{ false || github.repository == 'deckhouse/deckhouse' }}
needs:
- git_info
- doc_web_build
- main_web_build
# <template: web_links_test_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
# <template: werf_install_step>
- name: Install werf CLI
uses: werf/actions/install@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
# </template: werf_install_step>
- name: Prepare site structure
env:
WEB_REGISTRY_PATH: ${{steps.check_rw_registry.outputs.web_registry_path}}
run: |
type werf
werf version
BASEDIR=$(pwd)/docs
_TMPDIR=$(mktemp -d -t -p ${BASEDIR})
# Save TMPDIR to clean it later.
echo "_TMPDIR=$_TMPDIR" >> ${GITHUB_ENV}
echo "_TMPDIR=$_TMPDIR"
export WERF_REPO="${WEB_REGISTRY_PATH}"
echo -n 'use werf_repo '
echo $WERF_REPO | tr 'a-z' 'A-Z'
# Extract site content to the tmp directory.
export WERF_DIR=$BASEDIR/site
echo "Use werf_dir $WERF_DIR"
type werf && source $(werf ci-env github --verbose --as-file)
echo "werf stage image web-backend:"
werf stage image web-backend | tr 'a-z' 'A-Z'
echo "Run 'docker pull' from werf stage image web-backend"
docker pull $(werf stage image web-backend) || true
echo "werf stage image web-frontend:"
werf stage image web-frontend | tr 'a-z' 'A-Z'
echo "Run 'docker pull' from werf stage image web-frontend"
docker pull $(werf stage image web-frontend) || true
echo "Run 'docker cp' from werf stage image web-backend"
docker cp $(docker create --rm $(werf stage image web-backend)):/app/root ${_TMPDIR}/backend
echo "Run 'docker cp' from werf stage image web-frontend"
docker cp $(docker create --rm $(werf stage image web-frontend)):/app ${_TMPDIR}/frontend
# Extract documentation content to the tmp directory.
export WERF_DIR=$BASEDIR/documentation
type werf && source $(werf ci-env github --verbose --as-file)
echo "werf stage image web:"
werf stage image web | tr 'a-z' 'A-Z'
echo "Run 'docker pull' from werf stage image web"
docker pull $(werf stage image web)
echo "Run 'docker cp' from werf stage image web"
docker cp $(docker create --rm $(werf stage image web)):/app ${_TMPDIR}/documentation
# Create EN site structure.
echo "Create site structure in '${_TMPDIR}/site_en/'"
mkdir -p ${_TMPDIR}/site_en/ ${_TMPDIR}/site_ru/
touch ${_TMPDIR}/site_en/index.html ${_TMPDIR}/site_ru/index.html
rsync -a --exclude='ru' --exclude='en' --exclude='compare' ${_TMPDIR}/frontend/ ${_TMPDIR}/site_en/
rsync -a --exclude='ru' --exclude='en' --exclude='compare' ${_TMPDIR}/frontend/ ${_TMPDIR}/site_ru/
#
rsync -a ${_TMPDIR}/frontend/en/ ${_TMPDIR}/site_en/
rsync -a ${_TMPDIR}/frontend/ru/ ${_TMPDIR}/site_ru/
#
rsync -a ${_TMPDIR}/backend/en/ ${_TMPDIR}/site_en/
rsync -a ${_TMPDIR}/backend/ru/ ${_TMPDIR}/site_ru/
#
rsync -a --exclude='ru' --exclude='en' --exclude='compare' ${_TMPDIR}/documentation/ ${_TMPDIR}/site_en/documentation/
rsync -a --exclude='ru' --exclude='en' --exclude='compare' ${_TMPDIR}/documentation/ ${_TMPDIR}/site_ru/documentation/
rsync -a ${_TMPDIR}/documentation/en/ ${_TMPDIR}/site_en/documentation/
rsync -a ${_TMPDIR}/documentation/ru/ ${_TMPDIR}/site_ru/documentation/
#
rsync -a ${_TMPDIR}/documentation/{assets,css,images,js} ${_TMPDIR}/site_en/documentation
rsync -a ${_TMPDIR}/documentation/{assets,css,images,js} ${_TMPDIR}/site_ru/documentation
- name: Check links with html-proofer (EN)
run: |
# Do not exit on html-proofer error.
set +e
# Checking EN site
docker run --rm -v "${_TMPDIR}/site_en:/src:ro" klakegg/html-proofer:3.19.1 \
--allow-hash-href --check-html --empty-alt-ignore \
--url-ignore "/localhost/,/https\:\/\/t.me/,/docs-prv\.pcisecuritystandards\.org/,/gitlab.com\/profile/,/dash.cloudflare.com\/profile/,/example.com/,/vmware.com/,/.slack.com/,/habr.com/,/flant.ru/,/bcrypt-generator.com/,/candi\/bashible\/bashbooster/,/..\/..\/compare\//,/compare\/ru\//,/compare\/en\//,/\.yml$/,/\.yaml$/,/\.tmpl$/,/\.tpl$/" \
--url-swap "https\:\/\/deckhouse.io\/:/,\/documentation\/v1\/:/documentation/,\/documentation\/latest\/:/documentation/" \
--http-status-ignore "0,429" ${1}
# Emulate 'allow_failure: true' from Gitlab. Github has only two state: success and failure.
exit 0
- name: Check links with html-proofer (RU)
run: |
# Do not exit on html-proofer error.
set +e
# Checking RU site
docker run --rm -v "${_TMPDIR}/site_ru:/src:ro" klakegg/html-proofer:3.19.1 \
--allow-hash-href --check-html --empty-alt-ignore \
--url-ignore "/localhost/,/https\:\/\/t.me/,/docs-prv\.pcisecuritystandards\.org/,/gitlab.com\/profile/,/dash.cloudflare.com\/profile/,/example.com/,/vmware.com/,/.slack.com/,/habr.com/,/flant.ru/,/bcrypt-generator.com/,/candi\/bashible\/bashbooster/,/..\/..\/compare\//,/compare\/ru\//,/compare\/en\//,/\.yml$/,/\.yaml$/,/\.tmpl$/,/\.tpl$/" \
--url-swap "https\:\/\/deckhouse.io\/:/,\/documentation\/v1\/:/documentation/,\/documentation\/latest\/:/documentation/" \
--http-status-ignore "0,429" ${1}
# Emulate 'allow_failure: true' from Gitlab. Github has only two state: success and failure.
exit 0
- name: Clean TMPDIR
if: always()
run: |
if [[ -n $_TMPDIR ]] ; then
rm -rf $_TMPDIR
fi
# </template: web_links_test_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Web links test';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
validators:
name: Validators
needs:
- git_info
- build_fe
# <template: tests_template>
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_dev_registry_step>
- name: Check dev registry credentials
id: check_dev_registry
env:
HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to dev registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }}
logout: false
# </template: login_dev_registry_step>
# <template: login_rw_registry_step>
- name: Check rw registry credentials
id: check_rw_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to rw registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }}
logout: false
- name: Login to Github Container Registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }}
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_REGISTRY_USER }}
password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }}
logout: false
# </template: login_rw_registry_step>
- name: Run tests
env:
TESTS_IMAGE_NAME: ${{needs.build_fe.outputs.tests_image_name}}
run: |
if [[ -z ${TESTS_IMAGE_NAME} ]] ; then
echo "TESTS_IMAGE_NAME is empty"
exit 1
fi
# Decode image name from gzip+base64.
TESTS_IMAGE_NAME=$(echo ${TESTS_IMAGE_NAME} | base64 -d | gunzip)
# Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
echo "Tests image name: '${TESTS_IMAGE_NAME}'" | tr '[:lower:]' '[:upper:]'
echo "⚓️ 📥 [$(date -u)] Pull 'tests' image..."
docker pull ${TESTS_IMAGE_NAME}
echo "⚓️ 🏎 [$(date -u)] Run tests..."
docker run -w /deckhouse -v ~/go-pkg-cache:/go/pkg ${TESTS_IMAGE_NAME} go test -tags=validation -run Validation -timeout=${{env.TEST_TIMEOUT}} ./testing/...
# </template: tests_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Validators';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
deploy_latest_web:
name: Deploy latest doc and site
needs:
- git_info
- doc_web_build
- main_web_build
if: ${{ needs.git_info.outputs.ci_commit_ref_name == 'main' && github.repository == 'deckhouse/deckhouse' }}
runs-on: [self-hosted, regular]
steps:
# <template: started_at_output>
- name: Job started timestamp
id: started_at
run: |
unixTimestamp=$(date +%s)
echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT
# </template: started_at_output>
# <template: checkout_full_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
with:
fetch-depth: 0
# </template: checkout_full_step>
# <template: login_readonly_registry_step>
- name: Check readonly registry credentials
id: check_readonly_registry
env:
HOST: ${{secrets.DECKHOUSE_REGISTRY_READ_HOST}}
run: |
if [[ -n $HOST ]]; then
echo "has_credentials=true" >> $GITHUB_OUTPUT
echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_READ_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT
fi
- name: Login to readonly registry
uses: docker/login-action@v2.1.0
if: ${{ steps.check_readonly_registry.outputs.has_credentials == 'true' }}
with:
registry: ${{ secrets.DECKHOUSE_REGISTRY_READ_HOST }}
username: ${{ secrets.DECKHOUSE_REGISTRY_READ_USER }}
password: ${{ secrets.DECKHOUSE_REGISTRY_READ_PASSWORD }}
logout: false
# </template: login_readonly_registry_step>
# <template: doc_version_template>
- name: Set documentation version
env:
CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}}
run: |
echo "DOC_VERSION=${CI_COMMIT_TAG:-latest}" >> $GITHUB_ENV
# </template: doc_version_template>
# <template: deploy_doc_template>
- name: Deploy documentation to production
uses: werf/actions/converge@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
kube-config-base64-data: "${{ secrets.KUBECONFIG_BASE64_PROD }}"
env: web-production
env:
WERF_REPO: ${{ steps.check_readonly_registry.outputs.web_registry_path }}
WERF_DIR: "docs/documentation"
WERF_RELEASE: "deckhouse-doc-${{ env.DOC_VERSION }}"
WERF_NAMESPACE: deckhouse-web-production
WERF_SET_DOC_VERSION: "global.doc_version=${{ env.DOC_VERSION }}"
WERF_SET_URL: "global.url=deckhouse.io"
WERF_SET_URL_RU: "global.url_ru=deckhouse.ru"
WERF_SET_WEB_ENV: "web.env=web-production"
# </template: deploy_doc_template>
# <template: deploy_site_template>
- name: Deploy site to production
uses: werf/actions/converge@v1.2
with:
channel: ${{env.WERF_CHANNEL}}
kube-config-base64-data: "${{ secrets.KUBECONFIG_BASE64_PROD }}"
env: web-production
env:
WERF_REPO: ${{ steps.check_readonly_registry.outputs.web_registry_path }}
WERF_DIR: "docs/site"
WERF_RELEASE: "deckhouse-site"
WERF_NAMESPACE: deckhouse-web-production
WERF_SET_DOC_VERSION: "global.doc_version=${{ env.DOC_VERSION }}"
WERF_SET_ACTIVE_RELEASE: "global.active_release=v1"
WERF_SET_URL: "global.url=deckhouse.io"
WERF_SET_URL_RU: "global.url_ru=deckhouse.ru"
WERF_SET_WEB_ENV: "web.env=web-production"
WERF_SET_DOMAIN_MAP: "global.domain_map=eyJlbiIgOiAiZGVja2hvdXNlLmlvIiwgInJ1IiA6ICJkZWNraG91c2UucnUifQ=="
# </template: deploy_site_template>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'job,one-line';
const name = 'Deploy latest doc and site';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>
last_comment:
name: Update comment on finish
needs:
- started_at
- git_info
- go_generate
- workflow_render
- build_fe
- build_ee
- build_ce
- doc_web_build
- main_web_build
- deploy_latest_web
- tests
- matrix_tests
- dhctl_tests
- golangci_lint
- openapi_test_cases
- web_links_test
- validators
if: ${{ always() }}
runs-on: ubuntu-latest
env:
JOB_NAMES: |
{"build_ce":"Build CE","build_ee":"Build EE","build_fe":"Build FE","deploy_latest_web":"Deploy latest doc and site","dhctl_tests":"Dhctl Tests","doc_web_build":"Doc web build","go_generate":"Go Generate","golangci_lint":"GolangCI Lint","main_web_build":"Main web build","matrix_tests":"Matrix tests","openapi_test_cases":"OpenAPI Test Cases","tests":"Tests","validators":"Validators","web_links_test":"Web links test","workflow_render":"Render workflow"}
steps:
# <template: checkout_step>
- name: Checkout sources
uses: actions/checkout@v3.5.2
# </template: checkout_step>
# <template: update_comment_on_finish>
- name: Update comment on finish
id: update_comment_on_finish
if: ${{ always() && github.event_name == 'workflow_dispatch' && !!github.event.inputs.issue_number }}
env:
NEEDS_CONTEXT: ${{ toJSON(needs) }}
JOB_CONTEXT: ${{ toJSON(job) }}
STEPS_CONTEXT: ${{ toJSON(steps) }}
uses: actions/github-script@v6.4.1
with:
github-token: ${{secrets.BOATSWAIN_GITHUB_TOKEN}}
retries: 3
script: |
const statusConfig = 'workflow,final,restore-one-line';
const name = 'Build and test for release';
const needsContext = JSON.parse(process.env.NEEDS_CONTEXT);
const jobContext = JSON.parse(process.env.JOB_CONTEXT);
const stepsContext = JSON.parse(process.env.STEPS_CONTEXT);
let jobNames = null
if (process.env.JOB_NAMES) {
jobNames = JSON.parse(process.env.JOB_NAMES);
}
core.info(`needsContext: ${JSON.stringify(needsContext)}`);
core.info(`jobContext: ${JSON.stringify(jobContext)}`);
core.info(`stepsContext: ${JSON.stringify(stepsContext)}`);
core.info(`jobNames: ${JSON.stringify(jobNames)}`);
const ci = require('./.github/scripts/js/ci');
return await ci.updateCommentOnFinish({github, context, core, statusConfig, name, needsContext, jobContext, stepsContext, jobNames});
# </template: update_comment_on_finish>