From 1f6b35330f23e9561cc42130a49060a8042901d2 Mon Sep 17 00:00:00 2001 From: Roberto Giovanardi Date: Tue, 26 Nov 2024 10:47:12 +0100 Subject: [PATCH 1/2] ci: license publication automation A new .post job is run during a release, and is getting the previously generated license file, and is creating a PR over the mender-docs repository, to publish the licenses. Ticket: QA-820 Changelog: None Signed-off-by: Roberto Giovanardi --- .gitlab-ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ .licenses_header.md | 14 ++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 .licenses_header.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2bc2d3c..e4393bcd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,6 +62,14 @@ variables: description: "The repositorywhere to push images" value: "mendersoftware" + # Publish licenses + GITHUB_DOCS_REPO_URL: + description: "The Github Repo URL where to push the documentation" + value: "mendersoftware/mender-docs" + LICENSE_REMOTE_FILE: + description: "The changelog file in the remote changelog repo" + value: "302.Release-information/03.Open-source-licenses/01.Mender-Server/docs.md" + include: - project: "Northern.tech/Mender/mendertesting" file: @@ -410,6 +418,38 @@ publish:backend:licenses: paths: - backend/licenses.md +publish:backend:licenses:docs-site: + stage: .post + tags: + - hetzner-amd-beefy + rules: + # Only make available for stable branches + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/' + allow_failure: true + image: "registry.gitlab.com/northern.tech/mender/mender-test-containers:release-please-v1-master" + needs: + - job: publish:backend:licenses + artifacts: true + before_script: + # Setting up git + - git config --global user.email "${GITHUB_USER_EMAIL}" + - git config --global user.name "${GITHUB_USER_NAME}" + # GITHUB_TOKEN for Github cli authentication + - export GITHUB_TOKEN=${GITHUB_CLI_TOKEN} + script: + - git clone https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_DOCS_REPO_URL} + - cd ${GITHUB_DOCS_REPO_URL#*/} + - git checkout -b licenses-${CI_JOB_ID} + - cat ../.licenses_header.md > ${LICENSE_REMOTE_FILE} + - cat ../backend/licenses.md >> ${LICENSE_REMOTE_FILE} + - git add ${LICENSE_REMOTE_FILE} + - | + git commit -s -m "chore: add mender-server open source licenses" + - git push origin licenses-${CI_JOB_ID} + - gh pr create --title "${CI_COMMIT_TAG} Release - update Mender Server licenses" --body "Automated change to the Mender Server Licenses during ${CI_COMMIT_TAG} release" --base master --head licenses-${CI_JOB_ID} + after_script: + - git remote remove licenses-${CI_JOB_ID} + coveralls:done: image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/curlimages/curl stage: .post diff --git a/.licenses_header.md b/.licenses_header.md new file mode 100644 index 00000000..fb491dad --- /dev/null +++ b/.licenses_header.md @@ -0,0 +1,14 @@ +--- +title: Mender Server +taxonomy: + category: docs +shortcode-core: + active: false +--- + +All open source licenses used in Mender Server with the default build are shown below. + +Where copyright holder is missing, please refer to the repository of the software to identify the copyright holder(s). + +Third party licenses used in Mender Server: this file is autogenerated with `go list`. + From 6c84dc598ea796e7c8d6628067042582a0051e36 Mon Sep 17 00:00:00 2001 From: Manuel Zedel Date: Thu, 28 Nov 2024 13:23:14 +0100 Subject: [PATCH 2/2] ci: license publication automation for frontend added Ticket: QA-820 Changelog: None Signed-off-by: Manuel Zedel --- .gitlab-ci.yml | 32 +++++++++++++++-------------- frontend/pipeline.yml | 27 ++++++++++++++++++++++++ frontend/scripts/deno.json | 1 + frontend/scripts/formatLicenses.ts | 33 ++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 15 deletions(-) create mode 100644 frontend/scripts/formatLicenses.ts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4393bcd..9565e808 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -371,7 +371,7 @@ publish:backend:docker: - export MENDER_PUBLISH_TAG="${CI_COMMIT_REF_NAME}" script: - make -C backend -j 4 docker-publish NOASK=y \ - SKOPEO_ARGS='--digestfile '''${CI_PROJECT_DIR}'''/.digests/$(COMPONENT)' + SKOPEO_ARGS='--digestfile '''${CI_PROJECT_DIR}'''/.digests/$(COMPONENT)' - | if echo -n "${MENDER_PUBLISH_TAG}" | grep -E '^v[0-9]+\.v[0-9]+\.[0-9]+$'; then make -C backend -j 4 docker-publish NOASK=y \ @@ -418,7 +418,7 @@ publish:backend:licenses: paths: - backend/licenses.md -publish:backend:licenses:docs-site: +publish:licenses:docs-site: stage: .post tags: - hetzner-amd-beefy @@ -430,6 +430,8 @@ publish:backend:licenses:docs-site: needs: - job: publish:backend:licenses artifacts: true + - job: publish:frontend:licenses + artifacts: true before_script: # Setting up git - git config --global user.email "${GITHUB_USER_EMAIL}" @@ -442,6 +444,7 @@ publish:backend:licenses:docs-site: - git checkout -b licenses-${CI_JOB_ID} - cat ../.licenses_header.md > ${LICENSE_REMOTE_FILE} - cat ../backend/licenses.md >> ${LICENSE_REMOTE_FILE} + - cat ../frontend/licenses.md >> ${LICENSE_REMOTE_FILE} - git add ${LICENSE_REMOTE_FILE} - | git commit -s -m "chore: add mender-server open source licenses" @@ -458,7 +461,6 @@ coveralls:done: tags: - hetzner-amd-beefy - lint:commit: stage: lint needs: [] @@ -477,10 +479,11 @@ changelog: image: "registry.gitlab.com/northern.tech/mender/mender-test-containers:release-please-v1-master" stage: changelog variables: - GIT_DEPTH: 0 # Always get the full history - GIT_STRATEGY: clone # Always get the full history - GIT_CLIFF__BUMP__INITIAL_TAG: "4.0.0" # TODO: after the new tag is created, - # remove this variable + GIT_DEPTH: 0 # Always get the full history + GIT_STRATEGY: clone # Always get the full history + GIT_CLIFF__BUMP__INITIAL_TAG: + "4.0.0" # TODO: after the new tag is created, + # remove this variable tags: - hetzner-amd-beefy rules: @@ -496,12 +499,12 @@ changelog: - export GITHUB_TOKEN=${GITHUB_CLI_TOKEN} script: - release-please release-pr - --token=${GITHUB_BOT_TOKEN_REPO_FULL} - --repo-url=${GITHUB_REPO_URL} - --target-branch=${CI_COMMIT_REF_NAME} || echo "INFO - release already exists" # workaround because we shifted to prerelease versioning strategy and there's already a PR open + --token=${GITHUB_BOT_TOKEN_REPO_FULL} + --repo-url=${GITHUB_REPO_URL} + --target-branch=${CI_COMMIT_REF_NAME} || echo "INFO - release already exists" # workaround because we shifted to prerelease versioning strategy and there's already a PR open # git cliff: override the changelog - test $GIT_CLIFF == "false" && echo "INFO - Skipping git-cliff" && exit 0 - - git remote add github-${CI_JOB_ID} https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL} || true # Ignore already existing remote + - git remote add github-${CI_JOB_ID} https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL} || true # Ignore already existing remote - gh repo set-default https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL} - RELEASE_PLEASE_PR=$(gh pr list --author "${GITHUB_USER_NAME}" --head "release-please--branches--${CI_COMMIT_REF_NAME}" --json number | jq -r '.[0].number // empty') - test -z "$RELEASE_PLEASE_PR" && echo "No release-please PR found" && exit 0 @@ -534,9 +537,9 @@ release:github: - job: changelog script: - release-please github-release - --token=${GITHUB_BOT_TOKEN_REPO_FULL} - --repo-url=${GITHUB_REPO_URL} - --target-branch=${CI_COMMIT_REF_NAME} + --token=${GITHUB_BOT_TOKEN_REPO_FULL} + --repo-url=${GITHUB_REPO_URL} + --target-branch=${CI_COMMIT_REF_NAME} release:mender-docs-changelog: image: "registry.gitlab.com/northern.tech/mender/mender-test-containers:release-please-v1-master" @@ -663,7 +666,6 @@ release:mender-docs-changelog: - cd ${PROJECT_FOLDER} - rm -rf /tmp/helm - # # Mender Helm Rolling release # diff --git a/frontend/pipeline.yml b/frontend/pipeline.yml index 34829755..4455e9db 100644 --- a/frontend/pipeline.yml +++ b/frontend/pipeline.yml @@ -145,6 +145,11 @@ build:frontend:docker: --provenance false --push . + - docker run --rm --entrypoint "/bin/sh" -v $(pwd):/extract ${MENDER_IMAGE_GUI} -c "cp licenses.json /extract/" + artifacts: + expire_in: 1w + paths: + - frontend/licenses.json .template:test:frontend:acceptance: stage: test @@ -289,3 +294,25 @@ publish:frontend:docker: expire_in: 1w paths: - .digests + +publish:frontend:licenses: + stage: publish + tags: + - hetzner-amd-beefy + rules: + - changes: + paths: ['frontend/**/*'] + compare_to: '${RULES_CHANGES_COMPARE_TO_REF}' + - if: '$CI_COMMIT_REF_PROTECTED == "true"' + when: on_success + image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/denoland/deno:debian-2.0.2 + needs: + - job: build:frontend:docker + artifacts: true + script: + - deno task --cwd frontend/scripts licenseFormatting --rootDir $(pwd) + artifacts: + when: on_success + expire_in: 1w + paths: + - frontend/licenses.md diff --git a/frontend/scripts/deno.json b/frontend/scripts/deno.json index ea845639..af584a2b 100644 --- a/frontend/scripts/deno.json +++ b/frontend/scripts/deno.json @@ -2,6 +2,7 @@ "nodeModulesDir": "auto", "tasks": { "licenseCheck": "deno run --allow-read --allow-write --allow-run addLicense.js", + "licenseFormatting": "deno run --allow-read --allow-write formatLicenses.ts", "typeGeneration": "deno run --allow-env --allow-net --allow-read --allow-write typeGeneration/convert.js" } } diff --git a/frontend/scripts/formatLicenses.ts b/frontend/scripts/formatLicenses.ts new file mode 100644 index 00000000..e328fb7a --- /dev/null +++ b/frontend/scripts/formatLicenses.ts @@ -0,0 +1,33 @@ +import { join } from 'jsr:@std/path'; + +import { rootDir } from './common.js'; + +type License = { + name: string; + version: string; + author?: string; + repository: string; + source: string; + license: string; + licenseText: string; +}; + +const formatLicenseEntry = (licenseRecord: License) => { + const { name, version, repository, license, licenseText } = licenseRecord; + return `## ${name}\n +* Name: ${name} +* Version: ${version} +* License: [${license}](${repository})\n +\`\`\`\n${licenseText}\`\`\`\n`; +}; + +const processLicenseFile = async () => { + const root = rootDir ?? '.'; + const { default: licenses } = await import(join(root, 'frontend', 'licenses.json'), { + with: { type: 'json' } + }); + const licenseEntries = licenses.map(formatLicenseEntry).join('\n'); + await Deno.writeTextFile(join(root, 'frontend', 'licenses.md'), `# Licenses\n\n\n${licenseEntries}`); +}; + +await processLicenseFile();