pkg-auto: Add package automation scripts

pkg_auto/Makefile

@@ -0,0 +1,2 @@
+shellcheck:
+	docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --external-sources --check-sourced *.sh impl/*.sh

pkg_auto/README.md

@@ -0,0 +1,59 @@
+Scripts for package update automation
+=====================================
+
+A quick start from blank state:
+
+- clone scripts repo and create worktrees for the old state and new state (and optionally for the branch with package automation scripts if they are not a part of the `main` branch yet):
+  - `git clone https://github.com/flatcar/scripts.git scripts/main`
+  - `cd scripts/main`
+  - `PKG_AUTO="${PWD}/pkg_auto"`
+  - `git worktree add --branch weekly-updates ../weekly-updates origin/buildbot/weekly-portage-stable-package-updates-2024-09-23`
+  - if package automation is still not merged into the `main` branch then:
+    - `git worktree add --branch pkg-auto ../pkg-auto origin/krnowak/pkg-auto`
+    - `PKG_AUTO="${PWD}/../pkg-auto/pkg_auto"`
+- prepare for generating reports (create a directory, download necessary stuff, create config):
+  - `mkdir ../../weekly-updates`
+  - `cd ../../weekly-updates`
+  - `"${PKG_AUTO}/download_sdk_and_listings.sh" -s ../../scripts/main -x aux-cleanups aux`
+    - call `"${PKG_AUTO}/download_sdk_and_listings.sh" -h` to get help
+  - `"${PKG_AUTO}/generate_config.sh" -a aux -n weekly-updates -o main -r reports -s ../../scripts/main -x file,wd-cleanups config`
+    - call `"${PKG_AUTO}/generate_config.sh" -h` to get help
+- generate the reports:
+  - `"${PKG_AUTO}/generate_reports.sh" -w wd config`
+  - if the command above fails, the `reports` directory (see the `-r reports` flag in the call to  `generate_config.sh` above) will have some reports that may contain hints as to why the failure happened
+    - the `reports` directory may also contain files like `warnings` or `manual-work-needed`
+    - the items in `warnings` file should be addressed and the report generation should be rerun, see below
+    - the items in `manual-work-needed` are things to be done while processing the updates
+- in order to rerun the report generation, stuff from previous run should be removed beforehand:
+  - `source wd-cleanups`
+  - `rm -rf reports`
+  - `"${PKG_AUTO}/generate_reports.sh" -w wd config`
+- if generating reports succeeded, process the updates, update the PR with the changelogs and update summaries:
+  - this is the manual part, described below
+- after everything is done (like the PR got merged), things needs cleaning up:
+  - `source wd-cleanups`
+  - `rm -rf reports`
+  - `source aux-cleanups`
+
+Processing the updates (the manual part)
+========================================
+
+The generated directory with reports will contain the `updates` directory. Within there are two files: `summary_stubs` and `changelog_stubs`. The rest of the entries are categories and packages that were updated. The first file, `summary_stubs`, contains a list of packages that have changed and TODO items associated to each package. It is mostly used for being pasted into the pull request description as an aid for the reviewers. The latter, `changelog_stubs`, can serve as a base for changelog that should be added to the `scripts` repo.
+
+For each package in the `summary_stubs` there are TODO items. These are basically of four kinds:
+
+- to review the changes in the ebuild
+- to review the changes not in the ebuild (metadata, patch files)
+- to review the occurences of the package name in the scripts repository
+- to add a link to the release notes in case of a package update
+
+It is possible that none of the changes in the package are relevant to Flatcar (like when a package got stabilized for hppa, for instance), then the package should be just dropped from the `summary_stubs`. Note that the package update is relevant, so as such should stay in the file.
+
+The entries in `changelog_stubs` should be reviewed about relevancy (minor SDK-only packages should likely be dropped, they are seldom of interest to end-users) and the remaining entries should be updated with proper links to release notes.
+
+There may be also entries in `manual-work-needed` which may need addressing. Most often the reason is that the new package was added, or an existing package stopped being pulled in. This would need adding an entry to the `summary_stubs`.
+
+Other scripts
+=============
+
+There are other scripts in this directory. `inside_sdk_container.sh` is a script executed by `generate_reports.sh` inside the SDK to collect the package information. `sync_packages.sh` is a script that updates packages and saves the result to a new branch. `update_packages.sh` is `sync_packages.sh` + `generate_reports.sh`.

pkg_auto/config_template

@@ -0,0 +1,47 @@
+# Path to the toplevel directory of the scripts repo.
+scripts: ..
+
+# Path to the directory with auxiliary files.
+aux: ../../downloads
+
+# Path to the directory where update reports will be stored.
+reports: ../../reports
+
+# Base scripts branch for old state.
+#
+# Old state serves as an state before the updates.
+#
+# Optional, defaults to origin/main
+#old-base: origin/main
+
+# Base scripts branch for new state.
+#
+# New state serves as an state that's either after the updates or to-be-updated.
+#
+# Optional, defaults to old-base value
+#new-base: origin/main
+
+# Cleanups mode with semicolon-separated list of extra arguments.
+#
+# Can be:
+#
+# file,${path_to_file}
+# stores cleanup steps in the file. Can be later sourced to perform cleanups.
+#
+# trap
+# executes cleanups on script exit
+#
+# ignore
+# performs no cleanups at all
+#
+# Optional, if not specified, defaults to ignore.
+#cleanups: file,../../x-cleanups
+#cleanups: trap
+#cleanups: ignore
+
+# Override image name to use for an SDK container.
+#
+# Optional, defaults to
+# ghcr.io/flatcar/flatcar-sdk-all:${last_nightly_version_id_in_base}-${last_nightly_build_id_in_base}
+#amd64-sdk-img=
+#arm64-sdk-img=

pkg_auto/download_sdk_and_listings.sh

@@ -0,0 +1,265 @@
+#!/bin/bash
+
+##
+## Downloads package SDKs from bincache and loads them with
+## docker. Downloads package listings from bincache. Version can be
+## taken either from the latest nightly tag in the passed scripts
+## directory (with the -s option) or from specified version ID and
+## build ID (with -v and -b options). The results are written to the
+## passed downloads directory.
+##
+## Parameters:
+## -b <ID>: build ID, conflicts with -s
+## -h: this help
+## -s <DIR>: scripts repo directory, conflicts with -v and -b
+## -v <ID>: version ID, conflicts with -s
+## -nd: skip downloading of docker images
+## -p: download packages SDK images instead of the standard one (valid
+##     only when downloading docker images)
+## -nl: skip downloading of listings
+## -x <FILE>: cleanup file
+##
+## Positional:
+## 1: downloads directory
+##
+
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh"
+source "${PKG_AUTO_IMPL_DIR}/cleanups.sh"
+
+CLEANUP_FILE=
+SCRIPTS=
+VERSION_ID=
+BUILD_ID=
+SKIP_DOCKER=
+SKIP_LISTINGS=
+PKGS_SDK=
+
+while [[ ${#} -gt 0 ]]; do
+    case ${1} in
+        -b)
+            if [[ -n ${SCRIPTS} ]]; then
+                fail '-b cannot be used at the same time with -s'
+            fi
+            if [[ -z ${2:-} ]]; then
+                fail 'missing value for -b'
+            fi
+            BUILD_ID=${2}
+            shift 2
+            ;;
+        -h)
+            print_help
+            exit 0
+            ;;
+        -p)
+            PKGS_SDK=x
+            shift
+            ;;
+        -s)
+            if [[ -n ${VERSION_ID} ]] || [[ -n ${BUILD_ID} ]]; then
+                fail '-s cannot be used at the same time with -v or -b'
+            fi
+            if [[ -z ${2:-} ]]; then
+                fail 'missing value for -s'
+            fi
+            SCRIPTS=${2}
+            shift 2
+            ;;
+        -v)
+            if [[ -n ${SCRIPTS} ]]; then
+                fail '-v cannot be used at the same time with -s'
+            fi
+            if [[ -z ${2:-} ]]; then
+                fail 'missing value for -v'
+            fi
+            VERSION_ID=${2}
+            shift 2
+            ;;
+        -x)
+            if [[ -z ${2:-} ]]; then
+                fail 'missing value for -x'
+            fi
+            CLEANUP_FILE=${2}
+            shift 2
+            ;;
+        -nd)
+            SKIP_DOCKER=x
+            shift
+            ;;
+        -nl)
+            SKIP_LISTINGS=x
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        -*)
+            fail "unknown flag '${1}'"
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+if [[ ${#} -ne 1 ]]; then
+    fail 'Expected one positional parameter: a downloads directory'
+fi
+
+DOWNLOADS_DIR=$(realpath "${1}"); shift
+
+if [[ -z ${SCRIPTS} ]] && [[ -z ${VERSION_ID} ]]; then
+    fail 'need to pass either -s or -v (latter with the optional -b too)'
+fi
+
+if [[ -n ${CLEANUP_FILE} ]]; then
+    dirname_out "${CLEANUP_FILE}" cleanup_dir
+    # shellcheck disable=SC2154 # cleanup_dir is assigned in dirname_out
+    mkdir -p "${cleanup_dir}"
+    unset cleanup_dir
+    setup_cleanups file "${CLEANUP_FILE}"
+else
+    setup_cleanups ignore
+fi
+
+if [[ ! -d "${DOWNLOADS_DIR}" ]]; then
+    add_cleanup "rmdir ${DOWNLOADS_DIR@Q}"
+    mkdir "${DOWNLOADS_DIR}"
+fi
+
+function download() {
+    local url output
+    url="${1}"; shift
+    output="${1}"; shift
+
+    info "Downloading ${url}"
+    curl \
+        --fail \
+        --show-error \
+        --location \
+        --retry-delay 1 \
+        --retry 60 \
+        --retry-connrefused \
+        --retry-max-time 60 \
+        --connect-timeout 20 \
+        "${url}" >"${output}"
+}
+
+if [[ -n ${SCRIPTS} ]]; then
+    # shellcheck disable=SC1091 # sourcing generated file
+    VERSION_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}")
+    # shellcheck disable=SC1091 # sourcing generated file
+    BUILD_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}")
+fi
+
+ver_plus="${VERSION_ID}${BUILD_ID:++}${BUILD_ID}"
+ver_dash="${VERSION_ID}${BUILD_ID:+-}${BUILD_ID}"
+
+exts=(zst bz2 gz)
+
+# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds
+zst_cmds=(
+    zstd
+)
+
+# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds
+bz2_cmds=(
+    lbunzip2
+    pbunzip2
+    bunzip2
+)
+
+# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds
+gz_cmds=(
+    unpigz
+    gunzip
+)
+
+function download_sdk() {
+    local image_name=${1}; shift
+    local tarball_name=${1}; shift
+    local url_dir=${1}; shift
+
+    if docker images --format '{{.Repository}}:{{.Tag}}' | grep -q -x -F "${image_name}"; then
+        return 0
+    fi
+
+    info "No ${image_name} available in docker, pulling it from bincache"
+    local ext full_tarball_name tb
+    for ext in "${exts[@]}"; do
+        full_tarball_name="${tarball_name}.tar.${ext}"
+        tb="${DOWNLOADS_DIR}/${full_tarball_name}"
+        if [[ -s ${tb} ]]; then
+            break;
+        else
+            add_cleanup "rm -f ${tb@Q}"
+            if download "${url_dir}/${full_tarball_name}" "${tb}"; then
+                break
+            fi
+        fi
+    done
+    info "Loading ${image_name} into docker"
+    cmds_name="${ext}_cmds"
+    if ! declare -p "${cmds_name}" >/dev/null 2>/dev/null; then
+        fail "Failed to extract ${tb@Q} - no tools to extract ${ext@Q} files"
+    fi
+    declare -n cmds="${ext}_cmds"
+    loaded=
+    for cmd in "${cmds[@]}"; do
+        if ! command -v "${cmd}" >/dev/null; then
+            info "${cmd@Q} is not available"
+            continue
+        fi
+        info "Using ${cmd@Q} to extract the tarball"
+        "${cmd}" -d -c "${tb}" | docker load
+        add_cleanup "docker rmi ${image_name@Q}"
+        loaded=x
+        break
+    done
+    if [[ -z ${loaded} ]]; then
+        fail "Failed to extract ${tb@Q} - no known available tool to extract it"
+    fi
+    unset -n cmds
+}
+
+URL_DIR="https://bincache.flatcar-linux.net/containers/${ver_dash}"
+
+if [[ -z ${SKIP_DOCKER} ]] && [[ -z ${PKGS_SDK} ]]; then
+    download_sdk "ghcr.io/flatcar/flatcar-sdk-all:${ver_dash}" "flatcar-sdk-all-${ver_dash}" "${URL_DIR}"
+fi
+
+declare -a dsal_arches
+get_valid_arches dsal_arches
+
+for arch in "${dsal_arches[@]}"; do
+    if [[ -z ${SKIP_DOCKER} ]] && [[ -n ${PKGS_SDK} ]]; then
+        download_sdk "flatcar-packages-${arch}:${ver_dash}" "flatcar-packages-${arch}-${ver_dash}.tar.${ext}" "${URL_DIR}"
+    fi
+
+    if [[ -z ${SKIP_LISTINGS} ]]; then
+        listing_dir="${DOWNLOADS_DIR}/${arch}"
+        add_cleanup "rmdir ${listing_dir@Q}"
+        mkdir "${listing_dir}"
+        base_url="https://bincache.flatcar-linux.net/images/${arch}/${ver_plus}"
+
+        for infix in '' 'rootfs-included-sysexts'; do
+            index_html="${listing_dir}/${infix}${infix:+-}index.html"
+            url="${base_url}${infix:+/}${infix}"
+            add_cleanup "rm -f ${index_html@Q}"
+            download "${url}/" "${index_html}"
+
+            # get names of all files ending with _packages.txt
+            mapfile -t listing_files < <(grep -F '_packages.txt"' "${index_html}" | sed -e 's#.*"\(\./\)\?\([^"]*\)".*#\2#')
+
+            for listing in "${listing_files[@]}"; do
+                info "Downloading ${listing} for ${arch}"
+                listing_path="${listing_dir}/${listing}"
+                add_cleanup "rm -f ${listing_path@Q}"
+                download "${url}/${listing}" "${listing_path}"
+            done
+        done
+    fi
+done
+info 'Done'