From e34deffd31c29c41fb3bef4436ad338557a51508 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 12:45:35 -0700 Subject: [PATCH 01/15] Updating bootstratp to install Salt built Fedora packages. Initial support for installing on Photon via bootstrap script --- .github/workflows/templates/generate.py | 20 + bootstrap-salt.sh | 469 +++++++++++++++++++++++- kitchen.yml | 30 ++ 3 files changed, 518 insertions(+), 1 deletion(-) diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index f21ace903..7d5fe99c9 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -65,6 +65,8 @@ "opensuse-tumbleweed", "oraclelinux-7", "oraclelinux-8", + "photon-3", + "photon-4", "rockylinux-8", "ubuntu-2004", "ubuntu-2204", @@ -81,6 +83,8 @@ "debian-11", "oraclelinux-7", "oraclelinux-8", + "photon-3", + "photon-4", "rockylinux-8", "rockylinux-9", "ubuntu-2004", @@ -98,6 +102,8 @@ "debian-11", "oraclelinux-7", "oraclelinux-8", + "photon-3", + "photon-4", "rockylinux-8", "rockylinux-9", "ubuntu-2004", @@ -117,6 +123,8 @@ "gentoo-systemd", "opensuse-15", "opensuse-tumbleweed", + "photon-3", + "photon-4", "rockylinux-8", "rockylinux-9", "ubuntu-2204", @@ -137,6 +145,8 @@ "gentoo-systemd", "opensuse-15", "opensuse-tumbleweed", + "photon-3", + "photon-4", "rockylinux-8", "rockylinux-9", "ubuntu-2004", @@ -156,6 +166,8 @@ "gentoo-systemd", "opensuse-15", "opensuse-tumbleweed", + "photon-3", + "photon-4", "rockylinux-9", ] @@ -169,6 +181,8 @@ "gentoo-systemd", "opensuse-15", "opensuse-tumbleweed", + "photon-3", + "photon-4", ] BLACKLIST_GIT_3004 = [ @@ -190,6 +204,8 @@ "ubuntu-2110", "ubuntu-2204", "rockylinux-9", + "photon-3", + "photon-4", ] BLACKLIST_GIT_3005 = [ @@ -205,6 +221,8 @@ "gentoo-systemd", "opensuse-15", "opensuse-tumbleweed", + "photon-3", + "photon-4", "ubuntu-2004", "ubuntu-2110", "ubuntu-2204", @@ -327,6 +345,8 @@ "opensuse-tumbleweed": "Opensuse Tumbleweed", "oraclelinux-7": "Oracle Linux 7", "oraclelinux-8": "Oracle Linux 8", + "photon-3": "Photon OS 3", + "photon-4": "Photon OS 4", "rockylinux-8": "Rocky Linux 8", "rockylinux-9": "Rocky Linux 9", "ubuntu-2004": "Ubuntu 20.04", diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d3928058b..6b848f9b5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2153,6 +2153,15 @@ __dnf_install_noinput() { dnf -y install "${@}" || return $? } # ---------- end of function __dnf_install_noinput ---------- +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __tdnf_install_noinput +# DESCRIPTION: (DRY) dnf install with noinput options +#---------------------------------------------------------------------------------------------------------------------- +__tdnf_install_noinput() { + + tdnf -y install "${@}" || return $? +} # ---------- end of function __tdnf_install_noinput ---------- + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __git_clone_and_checkout # DESCRIPTION: (DRY) Helper function to clone and checkout salt to a @@ -4185,6 +4194,62 @@ install_debian_check_services() { # Fedora Install Functions # +__install_saltstack_fedora_onedir_repository() { + if [ "$ITYPE" = "stable" ]; then + repo_rev="$ONEDIR_REV" + else + repo_rev="latest" + fi + + __PY_VERSION_REPO="yum" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + # Avoid using '$releasever' variable for yum. + # Instead, this should work correctly on all RHEL variants. + base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/\$basearch/" + fi + gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" + + gpg_key_urls="" + for key in $gpg_key; do + gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") + done + + repo_file="/etc/yum.repos.d/salt.repo" + + if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + cat <<_eof > "$repo_file" +[saltstack] +name=SaltStack ${repo_rev} Release Channel for Fedora \$releasever +baseurl=${base_url} +skip_if_unavailable=True +gpgcheck=1 +gpgkey=${gpg_key_urls} +enabled=1 +enabled_metadata=1 +_eof + + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + fi + for key in $gpg_key; do + __rpm_import_gpg "${fetch_url}${key}" || return 1 + done + + yum clean metadata || return 1 + elif [ "$repo_rev" != "latest" ]; then + echowarn "salt.repo already exists, ignoring salt version argument." + echowarn "Use -F (forced overwrite) to install $repo_rev." + fi + + return 0 +} + install_fedora_deps() { if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then dnf -y update || return 1 @@ -4449,9 +4514,73 @@ install_fedora_check_services() { return 0 } + +install_fedora_onedir_deps() { + + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + yum -y update || return 1 + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ] && [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + echowarn "Detected -r or -R option while installing Salt packages for Python 3." + echowarn "Python 3 packages for older Salt releases requires the EPEL repository to be installed." + echowarn "Installing the EPEL repository automatically is disabled when using the -r or -R options." + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then + __install_saltstack_fedora_onedir_repository || return 1 + fi + + # If -R was passed, we need to configure custom repo url with rsync-ed packages + # Which is still handled in __install_saltstack_rhel_repository. This call has + # its own check in case -r was passed without -R. + if [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_fedora_onedir_repository || return 1 + fi + + if [ "$DISTRO_MAJOR_VERSION" -ge 8 ]; then + __PACKAGES="dnf-utils chkconfig" + else + __PACKAGES="yum-utils chkconfig" + fi + + __PACKAGES="${__PACKAGES} procps" + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __yum_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi + + return 0 + +} + + install_fedora_onedir() { STABLE_REV=$ONEDIR_REV - install_fedora_stable || return 1 + #install_fedora_stable || return 1 + + __PACKAGES="" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-cloud" + fi + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-master" + fi + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-minion" + fi + if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-syndic" + fi + + # shellcheck disable=SC2086 + __yum_install_noinput ${__PACKAGES} || return 1 return 0 } @@ -6738,6 +6867,344 @@ install_arch_linux_onedir_post() { # ####################################################################################################################### +####################################################################################################################### +# +# Photon OS Install Functions +# + +__install_saltstack_photon_onedir_repository() { + if [ "$ITYPE" = "stable" ]; then + repo_rev="$ONEDIR_REV" + else + repo_rev="latest" + fi + + __PY_VERSION_REPO="yum" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + # Avoid using '$releasever' variable for yum. + # Instead, this should work correctly on all RHEL variants. + base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/\$basearch/" + fi + gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" + + gpg_key_urls="" + for key in $gpg_key; do + gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") + done + + repo_file="/etc/yum.repos.d/salt.repo" + + if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + cat <<_eof > "$repo_file" +[saltstack] +name=SaltStack ${repo_rev} Release Channel for Photon \$releasever +baseurl=${base_url} +skip_if_unavailable=True +gpgcheck=1 +gpgkey=${gpg_key_urls} +enabled=1 +enabled_metadata=1 +_eof + + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" + if [ "${ONEDIR_REV}" = "nightly" ] ; then + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + fi + for key in $gpg_key; do + __rpm_import_gpg "${fetch_url}${key}" || return 1 + done + + #tdnf clean metadata || return 1 + elif [ "$repo_rev" != "latest" ]; then + echowarn "salt.repo already exists, ignoring salt version argument." + echowarn "Use -F (forced overwrite) to install $repo_rev." + fi + + return 0 +} + +install_photon_deps() { + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + tdnf -y update || return 1 + fi + + __PACKAGES="${__PACKAGES:=}" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -lt 3 ]; then + echoerror "There are no Python 2 stable packages for Fedora, only Py3 packages" + return 1 + fi + + PY_PKG_VER=3 + + __PACKAGES="${__PACKAGES} libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + fi + + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 + + return 0 +} + +install_photon_stable_post() { + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) + sleep 1 + systemctl daemon-reload + done +} + +install_photon_git_deps() { + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + # Packages are named python3- + PY_PKG_VER=3 + else + PY_PKG_VER=2 + fi + + __PACKAGES="" + if ! __check_command_exists ps; then + __PACKAGES="${__PACKAGES} procps-ng" + fi + if ! __check_command_exists git; then + __PACKAGES="${__PACKAGES} git" + fi + + if [ -n "${__PACKAGES}" ]; then + # shellcheck disable=SC2086 + __dnf_install_noinput ${__PACKAGES} || return 1 + __PACKAGES="" + fi + + __git_clone_and_checkout || return 1 + + if [ "${_POST_NEON_INSTALL}" -eq $BS_FALSE ]; then + + if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then + __PACKAGES="${__PACKAGES} ca-certificates" + fi + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-libcloud python${PY_PKG_VER}-netaddr" + fi + + install_photon_deps || return 1 + + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + if __check_command_exists python3; then + __python="python3" + fi + elif [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 2 ]; then + if __check_command_exists python2; then + __python="python2" + fi + else + if ! __check_command_exists python; then + echoerror "Unable to find a python binary?!" + return 1 + fi + # Let's hope it's the right one + __python="python" + fi + + grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" | while IFS=' + ' read -r dep; do + echodebug "Running '${__python}' -m pip install '${dep}'" + "${__python}" -m pip install "${dep}" || return 1 + done + else + __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" + if [ "${DISTRO_VERSION}" -ge 35 ]; then + __PACKAGES="${__PACKAGES} gcc-c++" + fi + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} || return 1 + fi + + # Let's trigger config_salt() + if [ "$_TEMP_CONFIG_DIR" = "null" ]; then + _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_photon_git() { + if [ "${_PY_EXE}" != "" ]; then + _PYEXE=${_PY_EXE} + echoinfo "Using the following python version: ${_PY_EXE} to install salt" + else + _PYEXE='python2' + fi + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then + ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 + else + ${_PYEXE} setup.py ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 + fi + return 0 +} + +install_photon_git_post() { + for fname in api master minion syndic; do + # Skip if not meant to be installed + [ $fname = "api" ] && \ + ([ "$_INSTALL_MASTER" -eq $BS_FALSE ] || ! __check_command_exists "salt-${fname}") && continue + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + # Account for new path for services files in later releases + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/common/salt-${fname}.service" ]; then + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/common" + else + _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm" + fi + __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + + # Salt executables are located under `/usr/local/bin/` on Fedora 36+ + #if [ "${DISTRO_VERSION}" -ge 36 ]; then + # sed -i -e 's:/usr/bin/:/usr/local/bin/:g' /lib/systemd/system/salt-*.service + #fi + + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) + sleep 1 + systemctl daemon-reload + done +} + +install_photon_restart_daemons() { + [ $_START_DAEMONS -eq $BS_FALSE ] && return + + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + systemctl stop salt-$fname > /dev/null 2>&1 + systemctl start salt-$fname.service && continue + echodebug "Failed to start salt-$fname using systemd" + if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then + systemctl status salt-$fname.service + journalctl -xe + fi + done +} + +install_photon_check_services() { + for fname in api master minion syndic; do + # Skip salt-api since the service should be opt-in and not necessarily started on boot + [ $fname = "api" ] && continue + + # Skip if not meant to be installed + [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue + [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue + + __check_services_systemd salt-$fname || return 1 + done + + return 0 +} + +install_photon_onedir_deps() { + + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + tdnf -y update || return 1 + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_TRUE" ] && [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + echowarn "Detected -r or -R option while installing Salt packages for Python 3." + echowarn "Python 3 packages for older Salt releases requires the EPEL repository to be installed." + echowarn "Installing the EPEL repository automatically is disabled when using the -r or -R options." + fi + + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ]; then + __install_saltstack_photon_onedir_repository || return 1 + fi + + # If -R was passed, we need to configure custom repo url with rsync-ed packages + # Which is still handled in __install_saltstack_rhel_repository. This call has + # its own check in case -r was passed without -R. + if [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_photon_onedir_repository || return 1 + fi + + __PACKAGES="procps-ng" + + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} || return 1 + + if [ "${_EXTRA_PACKAGES}" != "" ]; then + echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" + # shellcheck disable=SC2086 + __tdnf_install_noinput ${_EXTRA_PACKAGES} || return 1 + fi + + return 0 + +} + + +install_photon_onedir() { + STABLE_REV=$ONEDIR_REV + + __PACKAGES="" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-cloud" + fi + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-master" + fi + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + __PACKAGES="${__PACKAGES} salt-minion" + fi + if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then + __PACKAGES="${__PACKAGES} salt-syndic" + fi + + # shellcheck disable=SC2086 + __tdnf_install_noinput ${__PACKAGES} || return 1 + + return 0 +} + +install_photon_onedir_post() { + STABLE_REV=$ONEDIR_REV + install_photon_stable_post || return 1 + + return 0 +} +# +# Ended Fedora Install Functions +# +####################################################################################################################### + ####################################################################################################################### # # FreeBSD Install Functions diff --git a/kitchen.yml b/kitchen.yml index 615a8b267..7a0886327 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -146,6 +146,22 @@ platforms: - name: ubuntu-18.04 driver: run_command: /lib/systemd/systemd + - name: photon-3 + driver: + image: photon:3.0 + provision_command: + - tdnf -y install rpm procps-ng coreutils gawk + - echo "PubkeyAcceptedKeyTypes +ssh-rsa" | tee -a /etc/ssh/sshd_config + - sed -ie 's/PermitRootLogin no/PermitRootLogin yes/' /etc/ssh/sshd_config + - systemctl enable sshd.service + - name: photon-4 + driver: + image: photon:4.0 + provision_command: + - tdnf -y install rpm procps-ng coreutils gawk + - echo "PubkeyAcceptedKeyTypes +ssh-rsa" | tee -a /etc/ssh/sshd_config + - sed -ie 's/PermitRootLogin no/PermitRootLogin yes/' /etc/ssh/sshd_config + - systemctl enable sshd.service suites: - name: git-3003 @@ -194,6 +210,20 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 + - name: git-3006x + provisioner: + salt_version: 3006.x + salt_bootstrap_options: -x python3 -MPfq git %s + excludes: + - opensuse-15 + - opensuse-tumbleweed + - debian-11 + - arch + - gentoo + - gentoo-systemd + - freebsd-131 + - freebsd-123 + - openbsd-6 - name: stable-3003-0 provisioner: salt_version: 3003 From 5ef781774868ebc8e1a6bf91d4ce0b38e2b9dcb8 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 12:58:13 -0700 Subject: [PATCH 02/15] need to add photon-3 and photon-4 to list of Linux of distributions --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++ .github/workflows/templates/generate.py | 2 ++ 2 files changed, 32 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e81e9b3d1..cafabf014 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -504,6 +504,34 @@ jobs: instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' + photon-3: + name: Photon OS 3 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: photon-3 + display-name: Photon OS 3 + timeout: 20 + instances: '["stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + + + photon-4: + name: Photon OS 4 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: photon-4 + display-name: Photon OS 4 + timeout: 20 + instances: '["stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + + rockylinux-8: name: Rocky Linux 8 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -595,6 +623,8 @@ jobs: - opensuse-tumbleweed - oraclelinux-7 - oraclelinux-8 + - photon-3 + - photon-4 - rockylinux-8 - rockylinux-9 - ubuntu-2004 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 7d5fe99c9..b951f1aa4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -26,6 +26,8 @@ "opensuse-tumbleweed", "oraclelinux-7", "oraclelinux-8", + "photon-3", + "photon-4", "rockylinux-8", "rockylinux-9", "ubuntu-2004", From b5372f6e02e9cbc76b5e37cb1168075528213c11 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 13:28:31 -0700 Subject: [PATCH 03/15] removing distros from latest blacklist which have latest packages available. --- .github/workflows/ci.yml | 6 +++--- .github/workflows/templates/generate.py | 6 +----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cafabf014..ff0208f30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' amazon-2: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' debian-10: @@ -557,7 +557,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' ubuntu-2004: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index b951f1aa4..edc5cec58 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -321,11 +321,7 @@ "rockylinux-8", ] -LATEST_PKG_BLACKLIST = [ - "almalinux-9", - "centos-stream9", - "rockylinux-9", -] +LATEST_PKG_BLACKLIST = [] DISTRO_DISPLAY_NAMES = { "almalinux-8": "AlmaLinux 8", From a8b51e495a71b53b9048e570dc740414021cf99a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 14:22:13 -0700 Subject: [PATCH 04/15] fixing photon git installs, Using repo files for fedora and photon. --- bootstrap-salt.sh | 66 ++++++++++------------------------------------- 1 file changed, 14 insertions(+), 52 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6b848f9b5..4e96f4d5b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4206,39 +4206,20 @@ __install_saltstack_fedora_onedir_repository() { __PY_VERSION_REPO="py3" fi - # Avoid using '$releasever' variable for yum. - # Instead, this should work correctly on all RHEL variants. - base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}/" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/\$basearch/" - fi gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" - gpg_key_urls="" - for key in $gpg_key; do - gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") - done - repo_file="/etc/yum.repos.d/salt.repo" if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - cat <<_eof > "$repo_file" -[saltstack] -name=SaltStack ${repo_rev} Release Channel for Fedora \$releasever -baseurl=${base_url} -skip_if_unavailable=True -gpgcheck=1 -gpgkey=${gpg_key_urls} -enabled=1 -enabled_metadata=1 -_eof - - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" if [ "${ONEDIR_REV}" = "nightly" ] ; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" fi + + __fetch_url "${repo_file}" "${fetch_url}.repo" + for key in $gpg_key; do - __rpm_import_gpg "${fetch_url}${key}" || return 1 + __rpm_import_gpg "${fetch_url}/${key}" || return 1 done yum clean metadata || return 1 @@ -6884,42 +6865,23 @@ __install_saltstack_photon_onedir_repository() { __PY_VERSION_REPO="py3" fi - # Avoid using '$releasever' variable for yum. - # Instead, this should work correctly on all RHEL variants. - base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/\$basearch/" - fi - gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" - - gpg_key_urls="" - for key in $gpg_key; do - gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") - done - repo_file="/etc/yum.repos.d/salt.repo" if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - cat <<_eof > "$repo_file" -[saltstack] -name=SaltStack ${repo_rev} Release Channel for Photon \$releasever -baseurl=${base_url} -skip_if_unavailable=True -gpgcheck=1 -gpgkey=${gpg_key_urls} -enabled=1 -enabled_metadata=1 -_eof - - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" + fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" if [ "${ONEDIR_REV}" = "nightly" ] ; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" fi + + __fetch_url "${repo_file}" "${fetch_url}.repo" + + gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" + for key in $gpg_key; do - __rpm_import_gpg "${fetch_url}${key}" || return 1 + __rpm_import_gpg "${fetch_url}/${key}" || return 1 done - #tdnf clean metadata || return 1 + tdnf makecache || return 1 elif [ "$repo_rev" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $repo_rev." @@ -6989,7 +6951,7 @@ install_photon_git_deps() { if [ -n "${__PACKAGES}" ]; then # shellcheck disable=SC2086 - __dnf_install_noinput ${__PACKAGES} || return 1 + __tdnf_install_noinput ${__PACKAGES} || return 1 __PACKAGES="" fi From 7b735b155ec457b1d299944a3b54b8b77f8e98cd Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 15:27:29 -0700 Subject: [PATCH 05/15] blacklist Photon 3 and 4 from a couple git installs. --- .github/workflows/ci.yml | 30 ++++++++++++------------- .github/workflows/templates/generate.py | 6 +++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff0208f30..a68b83f7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "latest"]' amazon-2: @@ -277,7 +277,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' arch: @@ -291,7 +291,7 @@ jobs: distro-slug: arch display-name: Arch timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' centos-7: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "latest"]' debian-10: @@ -347,7 +347,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' debian-11: @@ -361,7 +361,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' fedora-35: @@ -375,7 +375,7 @@ jobs: distro-slug: fedora-35 display-name: Fedora 35 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' fedora-36: @@ -389,7 +389,7 @@ jobs: distro-slug: fedora-36 display-name: Fedora 36 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' fedora-37: @@ -431,7 +431,7 @@ jobs: distro-slug: gentoo display-name: Gentoo timeout: 90 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' gentoo-systemd: @@ -445,7 +445,7 @@ jobs: distro-slug: gentoo-systemd display-name: Gentoo (systemd) timeout: 90 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' opensuse-15: @@ -473,7 +473,7 @@ jobs: distro-slug: opensuse-tumbleweed display-name: Opensuse Tumbleweed timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-3006", "git-master", "latest"]' oraclelinux-7: @@ -529,7 +529,7 @@ jobs: distro-slug: photon-4 display-name: Photon OS 4 timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3006", "onedir-3006", "git-master", "latest"]' rockylinux-8: @@ -557,7 +557,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' + instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "latest"]' ubuntu-2004: @@ -571,7 +571,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' ubuntu-2204: @@ -585,7 +585,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' set-pipeline-exit-status: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index edc5cec58..9c9f8260f 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -257,12 +257,18 @@ "gentoo-systemd", "opensuse-15", "opensuse-tumbleweed", + "photon-3", + "photon-4", "rockylinux-9", "ubuntu-2004", "ubuntu-2110", "ubuntu-2204", ] +BLACKLIST_GIT_3006 = [ + "photon-4", +] + SALT_VERSIONS = [ "3003", "3004", From ac20b82f0d20e9f9e8fb93a89618187cf3ad47b6 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 16:23:31 -0700 Subject: [PATCH 06/15] removing extra GIT_VERSIONS_BLACKLIST --- .github/workflows/ci.yml | 30 ++++++++++++------------- .github/workflows/templates/generate.py | 4 ---- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a68b83f7a..0f4ed2335 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' amazon-2: @@ -277,7 +277,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' arch: @@ -291,7 +291,7 @@ jobs: distro-slug: arch display-name: Arch timeout: 20 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' centos-7: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' debian-10: @@ -347,7 +347,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' debian-11: @@ -361,7 +361,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' fedora-35: @@ -375,7 +375,7 @@ jobs: distro-slug: fedora-35 display-name: Fedora 35 timeout: 20 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' fedora-36: @@ -389,7 +389,7 @@ jobs: distro-slug: fedora-36 display-name: Fedora 36 timeout: 20 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' fedora-37: @@ -431,7 +431,7 @@ jobs: distro-slug: gentoo display-name: Gentoo timeout: 90 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' gentoo-systemd: @@ -445,7 +445,7 @@ jobs: distro-slug: gentoo-systemd display-name: Gentoo (systemd) timeout: 90 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' opensuse-15: @@ -473,7 +473,7 @@ jobs: distro-slug: opensuse-tumbleweed display-name: Opensuse Tumbleweed timeout: 20 - instances: '["git-3006", "git-master", "latest"]' + instances: '["git-master", "latest"]' oraclelinux-7: @@ -515,7 +515,7 @@ jobs: distro-slug: photon-3 display-name: Photon OS 3 timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3006", "onedir-3006", "git-master", "latest"]' photon-4: @@ -557,7 +557,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master", "latest"]' ubuntu-2004: @@ -571,7 +571,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' ubuntu-2204: @@ -585,7 +585,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "git-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' set-pipeline-exit-status: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 9c9f8260f..dd07957d1 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -265,10 +265,6 @@ "ubuntu-2204", ] -BLACKLIST_GIT_3006 = [ - "photon-4", -] - SALT_VERSIONS = [ "3003", "3004", From 2d646167cbe9399d157940603f618dbe3372f84d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 16:49:36 -0700 Subject: [PATCH 07/15] Blacklist Fedora-35 from latest so those are not available --- .github/workflows/ci.yml | 2 +- .github/workflows/templates/generate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f4ed2335..5f42d2fd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -375,7 +375,7 @@ jobs: distro-slug: fedora-35 display-name: Fedora 35 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["git-master"]' fedora-36: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index dd07957d1..e1208210d 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -323,7 +323,7 @@ "rockylinux-8", ] -LATEST_PKG_BLACKLIST = [] +LATEST_PKG_BLACKLIST = ["fedora-35"] DISTRO_DISPLAY_NAMES = { "almalinux-8": "AlmaLinux 8", From 725834f0b5a202dd14cea1b444f131f0a83004d6 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 16:52:02 -0700 Subject: [PATCH 08/15] removing Fedora 35, as it is EOL as of December 13, 2022. --- .github/workflows/ci.yml | 15 --------------- .github/workflows/templates/generate.py | 13 +------------ 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f42d2fd3..c68026895 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -364,20 +364,6 @@ jobs: instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' - fedora-35: - name: Fedora 35 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-35 - display-name: Fedora 35 - timeout: 20 - instances: '["git-master"]' - - fedora-36: name: Fedora 36 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -613,7 +599,6 @@ jobs: - centos-stream9 - debian-10 - debian-11 - - fedora-35 - fedora-36 - fedora-37 - fedora-38 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index e1208210d..e70412d9b 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -16,7 +16,6 @@ "centos-stream9", "debian-10", "debian-11", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -57,7 +56,6 @@ "centos-stream8", "debian-10", "debian-11", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -117,7 +115,6 @@ "arch", "centos-stream9", "debian-11", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -139,7 +136,6 @@ "centos-stream9", "debian-10", "debian-11", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -160,7 +156,6 @@ "almalinux-9", "arch", "centos-stream9", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -175,7 +170,6 @@ BLACKLIST_3005 = [ "arch", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -194,7 +188,6 @@ "centos-stream9", "debian-10", "debian-11", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -215,7 +208,6 @@ "arch", "debian-10", "debian-11", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -232,7 +224,6 @@ BLACKLIST_3006 = [ "arch", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -249,7 +240,6 @@ "centos-stream9", "debian-10", "debian-11", - "fedora-35", "fedora-36", "fedora-37", "fedora-38", @@ -323,7 +313,7 @@ "rockylinux-8", ] -LATEST_PKG_BLACKLIST = ["fedora-35"] +LATEST_PKG_BLACKLIST = [] DISTRO_DISPLAY_NAMES = { "almalinux-8": "AlmaLinux 8", @@ -335,7 +325,6 @@ "centos-stream9": "CentOS Stream 9", "debian-10": "Debian 10", "debian-11": "Debian 11", - "fedora-35": "Fedora 35", "fedora-36": "Fedora 36", "fedora-37": "Fedora 37", "fedora-38": "Fedora 38", From fedc18bfdfa82d572dc95ecec1a2f0d1dbbc29c1 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 21 Apr 2023 17:33:34 -0700 Subject: [PATCH 09/15] Do not run git-master on Photon 4 --- .github/workflows/ci.yml | 2 +- .github/workflows/templates/generate.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c68026895..d7330fcf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -515,7 +515,7 @@ jobs: distro-slug: photon-4 display-name: Photon OS 4 timeout: 20 - instances: '["stable-3006", "onedir-3006", "git-master", "latest"]' + instances: '["stable-3006", "onedir-3006", "latest"]' rockylinux-8: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index e70412d9b..427b65fe8 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -255,6 +255,10 @@ "ubuntu-2204", ] +BLACKLIST_GIT_MASTER = [ + "photon-4", +] + SALT_VERSIONS = [ "3003", "3004", @@ -597,6 +601,7 @@ def generate_test_jobs(): "3004": BLACKLIST_GIT_3004, "3005": BLACKLIST_GIT_3005, "3006": BLACKLIST_GIT_3006, + "master": BLACKLIST_GIT_MASTER, } # .0 versions are a virtual version for pinning to the first @@ -606,7 +611,7 @@ def generate_test_jobs(): continue if ( - salt_version in ("3003", "3004", "3005", "3006") + salt_version in ("3003", "3004", "3005", "3006", "master") and distro in BLACKLIST[salt_version] ): continue From fe9b906d0677395cca91d1962f8c0177d3b8b945 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 22 Apr 2023 10:39:19 -0700 Subject: [PATCH 10/15] making GPG_KEY and REPO_FILE uppercase. --- bootstrap-salt.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e23572aa4..0266cfc37 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4206,9 +4206,9 @@ __install_saltstack_fedora_onedir_repository() { __PY_VERSION_REPO="py3" fi - gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" + GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - repo_file="/etc/yum.repos.d/salt.repo" + REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" @@ -4216,9 +4216,9 @@ __install_saltstack_fedora_onedir_repository() { fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" fi - __fetch_url "${repo_file}" "${fetch_url}.repo" + __fetch_url "${REPO_FILE}" "${fetch_url}.repo" - for key in $gpg_key; do + for key in $GPG_KEY; do __rpm_import_gpg "${fetch_url}/${key}" || return 1 done @@ -6865,7 +6865,7 @@ __install_saltstack_photon_onedir_repository() { __PY_VERSION_REPO="py3" fi - repo_file="/etc/yum.repos.d/salt.repo" + REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" @@ -6873,11 +6873,11 @@ __install_saltstack_photon_onedir_repository() { fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" fi - __fetch_url "${repo_file}" "${fetch_url}.repo" + __fetch_url "${REPO_FILE}" "${fetch_url}.repo" - gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" + GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - for key in $gpg_key; do + for key in $GPG_KEY; do __rpm_import_gpg "${fetch_url}/${key}" || return 1 done From 5bdc4dbf11fa3a9082cb33405121514378f839af Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 22 Apr 2023 11:28:15 -0700 Subject: [PATCH 11/15] Fixing typos. --- bootstrap-salt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 0266cfc37..f7615dc53 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4210,7 +4210,7 @@ __install_saltstack_fedora_onedir_repository() { REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" if [ "${ONEDIR_REV}" = "nightly" ] ; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" @@ -6867,7 +6867,7 @@ __install_saltstack_photon_onedir_repository() { REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" if [ "${ONEDIR_REV}" = "nightly" ] ; then fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" From 432c6b6dfef490b40ebe0add0665ddeba7753b22 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 22 Apr 2023 11:29:36 -0700 Subject: [PATCH 12/15] Adding 3006 back in for Fedora --- .github/workflows/templates/generate.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 427b65fe8..96b89dbba 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -224,9 +224,6 @@ BLACKLIST_3006 = [ "arch", - "fedora-36", - "fedora-37", - "fedora-38", "gentoo", "gentoo-systemd", "opensuse-15", From 4b7652fdb4fbd10a64ca27fc21965be9a11418c3 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 22 Apr 2023 11:35:56 -0700 Subject: [PATCH 13/15] Adding 3006 onedir back in for Fedora --- .github/workflows/ci.yml | 6 +++--- .github/workflows/templates/generate.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7330fcf3..eecffb489 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -375,7 +375,7 @@ jobs: distro-slug: fedora-36 display-name: Fedora 36 timeout: 20 - instances: '["git-master", "latest"]' + instances: '["stable-3006", "onedir-3006", "git-master", "latest"]' fedora-37: @@ -389,7 +389,7 @@ jobs: distro-slug: fedora-37 display-name: Fedora 37 timeout: 20 - instances: '["latest"]' + instances: '["stable-3006", "onedir-3006", "latest"]' fedora-38: @@ -403,7 +403,7 @@ jobs: distro-slug: fedora-38 display-name: Fedora 38 timeout: 20 - instances: '["latest"]' + instances: '["stable-3006", "onedir-3006", "latest"]' gentoo: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 96b89dbba..886152806 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -81,6 +81,9 @@ "centos-stream9", "debian-10", "debian-11", + "fedora-36", + "fedora-37", + "fedora-38", "oraclelinux-7", "oraclelinux-8", "photon-3", @@ -576,10 +579,6 @@ def generate_test_jobs(): if distro not in STABLE_DISTROS: continue - if distro.startswith("fedora") and salt_version != "latest": - # Fedora does not keep old builds around - continue - if bootstrap_type == "git": if salt_version in GIT_VERSION_BLACKLIST: continue From e52d07754db5d7f8d254a1b85396c12f80457d41 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 22 Apr 2023 12:23:05 -0700 Subject: [PATCH 14/15] some fixes to ensure git install works on Photon, in particular Photon 4. --- .github/workflows/ci.yml | 2 +- .github/workflows/templates/generate.py | 4 +--- bootstrap-salt.sh | 10 ++++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eecffb489..8b24db506 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -515,7 +515,7 @@ jobs: distro-slug: photon-4 display-name: Photon OS 4 timeout: 20 - instances: '["stable-3006", "onedir-3006", "latest"]' + instances: '["stable-3006", "onedir-3006", "git-master", "latest"]' rockylinux-8: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 886152806..ba051048a 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -255,9 +255,7 @@ "ubuntu-2204", ] -BLACKLIST_GIT_MASTER = [ - "photon-4", -] +BLACKLIST_GIT_MASTER = [] SALT_VERSIONS = [ "3003", diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f7615dc53..ec4ce64a2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2687,7 +2687,7 @@ __activate_virtualenv() { # NAME: __install_pip_pkgs # DESCRIPTION: Return 0 or 1 if successfully able to install pip packages. Can provide a different python version to # install pip packages with. If $py_ver is not specified it will use the default python version. -# PARAMETERS: pkgs, py_ver +# PARAMETERS: pkgs, py_ver, upgrade #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { @@ -6992,13 +6992,15 @@ install_photon_git_deps() { done else __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" - if [ "${DISTRO_VERSION}" -ge 35 ]; then - __PACKAGES="${__PACKAGES} gcc-c++" - fi # shellcheck disable=SC2086 __tdnf_install_noinput ${__PACKAGES} || return 1 fi + # Need newer version of setuptools on Photon + _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION}" + echodebug "Running '${_PY_EXE} -m pip --upgrade install ${_setuptools_dep}'" + ${_PY_EXE} -m pip install --upgrade "${_setuptools_dep}" + # Let's trigger config_salt() if [ "$_TEMP_CONFIG_DIR" = "null" ]; then _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" From 5c59021c5d28cd2fc148d2333f4a78b25fb5f19b Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sun, 23 Apr 2023 19:57:49 -0700 Subject: [PATCH 15/15] uppercasing more variables --- bootstrap-salt.sh | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index ec4ce64a2..93d4bf12e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4196,9 +4196,9 @@ install_debian_check_services() { __install_saltstack_fedora_onedir_repository() { if [ "$ITYPE" = "stable" ]; then - repo_rev="$ONEDIR_REV" + REPO_REV="$ONEDIR_REV" else - repo_rev="latest" + REPO_REV="latest" fi __PY_VERSION_REPO="yum" @@ -4211,21 +4211,19 @@ __install_saltstack_fedora_onedir_repository() { REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" if [ "${ONEDIR_REV}" = "nightly" ] ; then - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" fi - __fetch_url "${REPO_FILE}" "${fetch_url}.repo" + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" - for key in $GPG_KEY; do - __rpm_import_gpg "${fetch_url}/${key}" || return 1 - done + __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 yum clean metadata || return 1 - elif [ "$repo_rev" != "latest" ]; then + elif [ "$REPO_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." - echowarn "Use -F (forced overwrite) to install $repo_rev." + echowarn "Use -F (forced overwrite) to install $REPO_REV." fi return 0 @@ -6855,9 +6853,9 @@ install_arch_linux_onedir_post() { __install_saltstack_photon_onedir_repository() { if [ "$ITYPE" = "stable" ]; then - repo_rev="$ONEDIR_REV" + REPO_REV="$ONEDIR_REV" else - repo_rev="latest" + REPO_REV="latest" fi __PY_VERSION_REPO="yum" @@ -6868,23 +6866,21 @@ __install_saltstack_photon_onedir_repository() { REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" if [ "${ONEDIR_REV}" = "nightly" ] ; then - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" fi - __fetch_url "${REPO_FILE}" "${fetch_url}.repo" + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - for key in $GPG_KEY; do - __rpm_import_gpg "${fetch_url}/${key}" || return 1 - done + __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 tdnf makecache || return 1 - elif [ "$repo_rev" != "latest" ]; then + elif [ "$REPO_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." - echowarn "Use -F (forced overwrite) to install $repo_rev." + echowarn "Use -F (forced overwrite) to install $REPO_REV." fi return 0