From fe7b3ae7ca9f556732ca4c1ce44b202c0feaa864 Mon Sep 17 00:00:00 2001 From: Shahid Shaikh Date: Wed, 28 Feb 2024 10:09:17 -0500 Subject: [PATCH 1/6] Add python and ruby support in runner image --- Dockerfile.almalinux | 17 ++++++++++++++--- Dockerfile.ubuntu | 17 ++++++++++++++--- build-files/build-image.sh | 30 +++++++++++++++++++++++++++++- build-selfhosted.sh | 9 +++++++-- setup-build-env.sh | 17 ++++++++++++++--- 5 files changed, 78 insertions(+), 12 deletions(-) diff --git a/Dockerfile.almalinux b/Dockerfile.almalinux index 892da18..07982e3 100644 --- a/Dockerfile.almalinux +++ b/Dockerfile.almalinux @@ -1,6 +1,6 @@ FROM almalinux:9 -ARG RUNNERREPO="https://github.com/actions/runner" RUNNERPATCH SDK ARCH +ARG RUNNERREPO="https://github.com/actions/runner" RUNNERPATCH SDK ARCH BTOOLS RUN dnf update -y -q && \ dnf install -y -q wget git which langpacks-en glibc-all-langpacks sudo @@ -39,9 +39,20 @@ RUN mkdir -p /opt/runner && \ chown -R almalinux:almalinux /opt/runner && \ su -c "/opt/runner/config.sh --version" almalinux -RUN dnf install -y -q cmake make automake autoconf m4 gcc gcc-c++ libtool epel-release +RUN dnf install -y -q cmake make automake autoconf m4 gcc gcc-c++ libtool epel-release zlib-devel libffi-devel libyaml -RUN rm -rf /tmp/runner /var/cache/dnf/* /tmp/runner.patch && \ +COPY build-files/install-python.sh /tmp +COPY build-files/install-ruby.sh /tmp + +RUN if [ "${BTOOLS}" -eq 1 ]; then \ + chmod +x /tmp/install-python.sh && \ + /tmp/install-python.sh && \ + chmod +x /tmp/install-ruby.sh && \ + /tmp/install-ruby.sh && \ + chown -R almalinux:almalinux /opt/runner; \ + fi + +RUN rm -rf /tmp/runner /var/cache/dnf/* /tmp/runner.patch /tmp/install-python.sh /tmp/install-ruby.sh && \ dnf clean all USER almalinux diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 343faeb..c5c0b89 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -1,6 +1,6 @@ FROM ubuntu:22.04 -ARG RUNNERREPO="https://github.com/actions/runner" RUNNERPATCH SDK ARCH +ARG RUNNERREPO="https://github.com/actions/runner" RUNNERPATCH SDK ARCH BTOOLS ENV DEBIAN_FRONTEND=noninteractive @@ -47,9 +47,20 @@ RUN mkdir -p /opt/runner && \ chown -R ubuntu:ubuntu /opt/runner && \ su -c "/opt/runner/config.sh --version" ubuntu -RUN apt-get -qq -y install cmake make automake autoconf m4 gcc-12-base libtool +RUN apt-get -qq -y install cmake make automake autoconf m4 gcc-12-base libtool zlib1g-dev libffi-dev libyaml-dev -RUN rm -rf /tmp/runner /tmp/runner.patch /tmp/convert-rpm.sh +COPY build-files/install-python.sh /tmp +COPY build-files/install-ruby.sh /tmp + +RUN if [ "${BTOOLS}" -eq 1 ]; then \ + chmod +x /tmp/install-python.sh && \ + /tmp/install-python.sh && \ + chmod +x /tmp/install-ruby.sh && \ + /tmp/install-ruby.sh && \ + chown -R ubuntu:ubuntu /opt/runner; \ + fi + +RUN rm -rf /tmp/runner /tmp/runner.patch /tmp/convert-rpm.sh /tmp/install-python.sh /tmp/install-ruby.sh USER ubuntu diff --git a/build-files/build-image.sh b/build-files/build-image.sh index d5f9567..8e8c1a6 100755 --- a/build-files/build-image.sh +++ b/build-files/build-image.sh @@ -7,6 +7,9 @@ update_fresh_container() { if [ $? -ne 0 ]; then exit 32 fi + if [ ${BTOOLS} -eq 1 ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get install wget gcc make zlib1g-dev libffi-dev libtool libyaml-dev -y + fi sudo apt autoclean echo "Initializing LXD environment" @@ -124,6 +127,22 @@ install_runner() { return $? } +install_python() { + echo "Installing Python" + chmod +x /home/ubuntu/install-python.sh + sudo /home/ubuntu/install-python.sh + sudo chown ubuntu:ubuntu -R /opt/runner + return $? +} + +install_ruby() { + echo "Installing Ruby" + chmod +x /home/ubuntu/install-ruby.sh + sudo /home/ubuntu/install-ruby.sh + sudo chown ubuntu:ubuntu -R /opt/runner + return $? +} + cleanup() { rm -rf /home/ubuntu/build-image.sh /home/ubuntu/runner-${ARCH}.patch \ /tmp/runner /tmp/preseed-yaml @@ -142,6 +161,11 @@ run() { if [ ${RC} -eq 0 ]; then install_runner RC=$? + if [ ${BTOOLS} -eq 1 ]; then + install_python + install_ruby + RC=$? + fi fi fi fi @@ -153,7 +177,8 @@ export HOME=/home/ubuntu ARCH=`uname -m` SDK="" RUNNERREPO="https://github.com/actions/runner" -while getopts "a:s:" opt +BTOOLS="0" +while getopts "a:s:t:" opt do case ${opt} in a) @@ -162,6 +187,9 @@ do s) SDK=${OPTARG} ;; + t) + BTOOLS=${OPTARG} + ;; *) exit 4 ;; diff --git a/build-selfhosted.sh b/build-selfhosted.sh index e99fffa..095727f 100755 --- a/build-selfhosted.sh +++ b/build-selfhosted.sh @@ -5,6 +5,7 @@ usage() { echo "Where flags:" echo "-b [docker|podman] Image build tool to use - defaults to which it finds first" echo "-s SDK to use (6 or 7 ...). Default 6 for s390x and 7 for ppc64le" + echo "-t [1|0] Include tools/compilers - defaults to 0" echo "-h Display this usage information" echo echo "If no distribution is specified then images for both are built" @@ -14,6 +15,7 @@ usage() { ARCH=`uname -m` DISTROS="" BUILDER=`which podman 2>/dev/null` +BTOOLS="0" if [ -z ${BUILDER} ]; then BUILDER=`which docker 2>/dev/null` fi @@ -21,7 +23,7 @@ if [ -z ${BUILDER} ]; then echo "Need podman or docker installed" >&2 exit 1 fi -while getopts "b:hs:" opt +while getopts "b:ht:s:" opt do case "${opt}" in b) @@ -30,6 +32,9 @@ do h) usage ;; + t) + BTOOLS="${OPTARG}" + ;; s) SDK="${OPTARG}" ;; @@ -57,6 +62,6 @@ do echo "${dist} not supported" >&2 else ${BUILDER} build -f Dockerfile.${dist} --build-arg RUNNERPATCH=build-files/runner-${ARCH}.patch \ - --build-arg SDK=${SDK} --build-arg ARCH=${ARCH} --tag runner:${dist} . + --build-arg SDK=${SDK} --build-arg ARCH=${ARCH} --build-arg BTOOLS=${BTOOLS} --tag runner:${dist} . fi done diff --git a/setup-build-env.sh b/setup-build-env.sh index 3c11250..2c7ad70 100755 --- a/setup-build-env.sh +++ b/setup-build-env.sh @@ -11,6 +11,7 @@ usage() { echo "-s .NET SDK level" echo " - Defaults to value in build script for ppc64le" echo " - Ignored for s390x which uses an RPM" + echo "-t <1|0> - Include build tools/compilers in image. Defaults to 0." echo "-h Display this usage information" exit } @@ -75,8 +76,14 @@ build_image_in_container() { echo "Copy the gha-service unit file into gha-builder" lxc file push ${BUILD_PREREQS_PATH}/gha-runner.service "${BUILD_CONTAINER}/etc/systemd/system/gha-runner.service" + echo "Copy the install-python script into gha-builder" + lxc file push --mode 0755 "${BUILD_PREREQS_PATH}/install-python.sh" "${BUILD_CONTAINER}${BUILD_HOME}/install-python.sh" + + echo "Copy the install-ruby script into gha-builder" + lxc file push --mode 0755 "${BUILD_PREREQS_PATH}/install-ruby.sh" "${BUILD_CONTAINER}${BUILD_HOME}/install-ruby.sh" + echo "Running build-image.sh" - lxc exec "${BUILD_CONTAINER}" --user 1000 --group 1000 -- ${BUILD_HOME}/build-image.sh -a ${ACTION_RUNNER} ${SDK} + lxc exec "${BUILD_CONTAINER}" --user 1000 --group 1000 -- ${BUILD_HOME}/build-image.sh -a ${ACTION_RUNNER} -s ${SDK} -t ${BTOOLS} RC=$? if [ ${RC} -eq 0 ]; then @@ -113,6 +120,7 @@ prolog() { export ACTION_RUNNER="https://github.com/actions/runner" export EXPORT="distro/lxc-runner" export SDK="" + export BTOOLS="0" export OS_NAME="${OS_NAME:-ubuntu}" export OS_VERSION="${OS_VERSION:-22.04}" @@ -129,7 +137,7 @@ prolog() { } prolog -while getopts "a:o:hs:" opt +while getopts "a:o:ht:s:" opt do case "${opt}" in a) @@ -141,8 +149,11 @@ do h) usage ;; + t) + BTOOLS="${OPTARG}" + ;; s) - SDK="-s ${OPTARG}" + SDK="${OPTARG}" ;; *) usage From fe21a51c2449eb149af3312dfdc44cf184651ec8 Mon Sep 17 00:00:00 2001 From: Shahid Shaikh Date: Mon, 4 Mar 2024 15:39:00 -0500 Subject: [PATCH 2/6] Add dotnet support in runner image for build-selfhosted.sh --- build-files/install-python.sh | 27 +++++++++++++++++++++++++++ build-files/install-ruby.sh | 19 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 build-files/install-python.sh create mode 100644 build-files/install-ruby.sh diff --git a/build-files/install-python.sh b/build-files/install-python.sh new file mode 100644 index 0000000..86fc93e --- /dev/null +++ b/build-files/install-python.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +PYTHON_VERSIONS="3.8.18 3.10.13 3.12.1" + +for pyver in ${PYTHON_VERSIONS} +do + export PYTHON_VERSION=${pyver} + export PYTHON_MAJOR=${PYTHON_VERSION%.*.*} + wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz + tar -xzf Python-${PYTHON_VERSION}.tgz + cd Python-${PYTHON_VERSION} + ./configure --prefix=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le --enable-shared --enable-optimizations --enable-ipv6 LDFLAGS=-Wl,-rpath=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le/lib,--disable-new-dtags + make -j$(nproc) + sudo make install + sudo touch /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le.complete + sudo ln -s /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le/bin/python${PYTHON_MAJOR} /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le/bin/python + cd .. + rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz +done + + + + + + + + diff --git a/build-files/install-ruby.sh b/build-files/install-ruby.sh new file mode 100644 index 0000000..0615e49 --- /dev/null +++ b/build-files/install-ruby.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +sudo apt-get install -y gcc g++ make zlib1g-dev libffi-dev libtool libyaml-dev libssl-dev openjdk-11-jdk ruby + +wget -O /tmp/ruby-build-v20240119.tar.gz https://github.com/rbenv/ruby-build/archive/refs/tags/v20240119.tar.gz +cd /tmp +tar -xzf ruby-build-*.tar.gz +sudo PREFIX=/usr/local ./ruby-build-*/install.sh + +ruby-build --list| while IFS= read -r line; do +if [[ "$line" != *"picoruby"* ]] && [[ "$line" != *"truffleruby"* ]]; then + ruby-build $line /opt/runner/_work/_tool/Ruby/$line/ppc64 + touch /opt/runner/_work/_tool/Ruby/$line/ppc64.complete + echo "Installed $line"; +fi +done + +rm -rf /tmp/ruby-build-v20240119.tar.gz ./ruby-build-* + From 0d4739312eff73a60a0a5de006c909cbef4db3fd Mon Sep 17 00:00:00 2001 From: Shahid Shaikh Date: Thu, 14 Mar 2024 16:38:48 -0400 Subject: [PATCH 3/6] Incorporate code review comments --- Dockerfile.almalinux | 14 +++++++++----- Dockerfile.ubuntu | 11 ++++++----- build-files/build-image.sh | 3 --- build-files/install-python.sh | 29 ++++++++++++++++++----------- build-files/install-ruby.sh | 24 ++++++++++++++++++++---- 5 files changed, 53 insertions(+), 28 deletions(-) diff --git a/Dockerfile.almalinux b/Dockerfile.almalinux index 07982e3..d90d307 100644 --- a/Dockerfile.almalinux +++ b/Dockerfile.almalinux @@ -4,6 +4,9 @@ ARG RUNNERREPO="https://github.com/actions/runner" RUNNERPATCH SDK ARCH BTOO RUN dnf update -y -q && \ dnf install -y -q wget git which langpacks-en glibc-all-langpacks sudo + +RUN dnf install -y -q dnf-plugins-core && \ + dnf config-manager --set-enabled crb RUN dnf install -y -q dotnet-sdk-${SDK}.0 && \ echo "Using SDK - `dotnet --version`" @@ -39,16 +42,17 @@ RUN mkdir -p /opt/runner && \ chown -R almalinux:almalinux /opt/runner && \ su -c "/opt/runner/config.sh --version" almalinux -RUN dnf install -y -q cmake make automake autoconf m4 gcc gcc-c++ libtool epel-release zlib-devel libffi-devel libyaml +RUN dnf install -y -q cmake make automake autoconf m4 gcc gcc-c++ libtool epel-release COPY build-files/install-python.sh /tmp COPY build-files/install-ruby.sh /tmp RUN if [ "${BTOOLS}" -eq 1 ]; then \ - chmod +x /tmp/install-python.sh && \ - /tmp/install-python.sh && \ - chmod +x /tmp/install-ruby.sh && \ - /tmp/install-ruby.sh && \ + dnf install -y -q java-11-openjdk openssl-devel ruby perl; \ + chmod +x /tmp/install-python.sh; \ + /tmp/install-python.sh; \ + chmod +x /tmp/install-ruby.sh; \ + /tmp/install-ruby.sh; \ chown -R almalinux:almalinux /opt/runner; \ fi diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index c5c0b89..dffa13d 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -47,16 +47,17 @@ RUN mkdir -p /opt/runner && \ chown -R ubuntu:ubuntu /opt/runner && \ su -c "/opt/runner/config.sh --version" ubuntu -RUN apt-get -qq -y install cmake make automake autoconf m4 gcc-12-base libtool zlib1g-dev libffi-dev libyaml-dev +RUN apt-get -qq -y install cmake make automake autoconf m4 gcc-12-base libtool COPY build-files/install-python.sh /tmp COPY build-files/install-ruby.sh /tmp RUN if [ "${BTOOLS}" -eq 1 ]; then \ - chmod +x /tmp/install-python.sh && \ - /tmp/install-python.sh && \ - chmod +x /tmp/install-ruby.sh && \ - /tmp/install-ruby.sh && \ + apt-get -qq -y install openjdk-11-jdk libssl-dev ruby perl; \ + chmod +x /tmp/install-python.sh; \ + /tmp/install-python.sh; \ + chmod +x /tmp/install-ruby.sh; \ + /tmp/install-ruby.sh; \ chown -R ubuntu:ubuntu /opt/runner; \ fi diff --git a/build-files/build-image.sh b/build-files/build-image.sh index 8e8c1a6..a07c26c 100755 --- a/build-files/build-image.sh +++ b/build-files/build-image.sh @@ -7,9 +7,6 @@ update_fresh_container() { if [ $? -ne 0 ]; then exit 32 fi - if [ ${BTOOLS} -eq 1 ]; then - sudo DEBIAN_FRONTEND=noninteractive apt-get install wget gcc make zlib1g-dev libffi-dev libtool libyaml-dev -y - fi sudo apt autoclean echo "Initializing LXD environment" diff --git a/build-files/install-python.sh b/build-files/install-python.sh index 86fc93e..4176072 100644 --- a/build-files/install-python.sh +++ b/build-files/install-python.sh @@ -1,6 +1,21 @@ #!/bin/bash +if [ -f /etc/os-release ]; then + ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') + case $ID in + almalinux|centos|rhel) + sudo dnf install -y -q wget make gcc-c++ libtool zlib-devel libffi-devel libyaml-devel;; + ubuntu) + sudo apt-get install -y wget gcc g++ make libtool zlib1g-dev libffi-dev libyaml-dev;; + *) exit 1;; + esac +else + echo "Unknown OS distribution" + exit 1 +fi + PYTHON_VERSIONS="3.8.18 3.10.13 3.12.1" +M_ARCH=$(uname -m) for pyver in ${PYTHON_VERSIONS} do @@ -9,19 +24,11 @@ do wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz tar -xzf Python-${PYTHON_VERSION}.tgz cd Python-${PYTHON_VERSION} - ./configure --prefix=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le --enable-shared --enable-optimizations --enable-ipv6 LDFLAGS=-Wl,-rpath=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le/lib,--disable-new-dtags + ./configure --prefix=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH} --enable-shared --enable-optimizations --enable-ipv6 LDFLAGS=-Wl,-rpath=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/lib,--disable-new-dtags make -j$(nproc) sudo make install - sudo touch /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le.complete - sudo ln -s /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le/bin/python${PYTHON_MAJOR} /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/ppc64le/bin/python + sudo touch /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}.complete + sudo ln -s /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/bin/python${PYTHON_MAJOR} /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/bin/python cd .. rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz done - - - - - - - - diff --git a/build-files/install-ruby.sh b/build-files/install-ruby.sh index 0615e49..0d5b12e 100644 --- a/build-files/install-ruby.sh +++ b/build-files/install-ruby.sh @@ -1,6 +1,23 @@ #!/bin/bash -sudo apt-get install -y gcc g++ make zlib1g-dev libffi-dev libtool libyaml-dev libssl-dev openjdk-11-jdk ruby +if [ -f /etc/os-release ]; then + ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') + case $ID in + almalinux|centos|rhel) + sudo dnf install -y -q wget make gcc-c++ libtool zlib-devel libffi-devel libyaml-devel openssl-devel java-11-openjdk ruby rubygem-rake perl;; + ubuntu) + sudo apt-get install -y wget gcc g++ make libtool zlib1g-dev libffi-dev libyaml-dev libssl-dev openjdk-11-jdk ruby;; + *) exit 0;; + esac +else + echo "Unknown OS distribution" + exit 0 +fi + +M_ARCH=$(uname -m) +if [ "${M_ARCH}" = "ppc64le" ]; then + M_ARCH=ppc64 +fi wget -O /tmp/ruby-build-v20240119.tar.gz https://github.com/rbenv/ruby-build/archive/refs/tags/v20240119.tar.gz cd /tmp @@ -9,11 +26,10 @@ sudo PREFIX=/usr/local ./ruby-build-*/install.sh ruby-build --list| while IFS= read -r line; do if [[ "$line" != *"picoruby"* ]] && [[ "$line" != *"truffleruby"* ]]; then - ruby-build $line /opt/runner/_work/_tool/Ruby/$line/ppc64 - touch /opt/runner/_work/_tool/Ruby/$line/ppc64.complete + ruby-build $line /opt/runner/_work/_tool/Ruby/$line/${M_ARCH} + touch /opt/runner/_work/_tool/Ruby/$line/${M_ARCH}.complete echo "Installed $line"; fi done rm -rf /tmp/ruby-build-v20240119.tar.gz ./ruby-build-* - From b668dd33e8187fb61b742fced7bf1f47467de5de Mon Sep 17 00:00:00 2001 From: Shahid Shaikh Date: Sat, 16 Mar 2024 08:34:58 -0400 Subject: [PATCH 4/6] Fix issues with Python --- build-files/install-python.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/build-files/install-python.sh b/build-files/install-python.sh index 4176072..aa0856d 100644 --- a/build-files/install-python.sh +++ b/build-files/install-python.sh @@ -1,22 +1,24 @@ #!/bin/bash +PYTHON_VERSIONS="3.8.18 3.10.13 3.12.1" +M_ARCH=$(uname -m) + if [ -f /etc/os-release ]; then ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') case $ID in almalinux|centos|rhel) - sudo dnf install -y -q wget make gcc-c++ libtool zlib-devel libffi-devel libyaml-devel;; + sudo dnf install -y -q wget gcc-c++ make libtool zlib-devel libffi-devel libyaml-devel libxml2-devel libxslt-devel openssl-devel sqlite-devel;; ubuntu) - sudo apt-get install -y wget gcc g++ make libtool zlib1g-dev libffi-dev libyaml-dev;; - *) exit 1;; + sudo apt-get install -y wget gcc g++ make libtool zlib1g-dev libffi-dev libyaml-dev libxml2-dev libxslt1-dev libssl-dev libsqlite3-dev;; + *) + echo "This OS is not supported." + exit 1;; esac else - echo "Unknown OS distribution" + echo "Unknown OS distribution." exit 1 fi -PYTHON_VERSIONS="3.8.18 3.10.13 3.12.1" -M_ARCH=$(uname -m) - for pyver in ${PYTHON_VERSIONS} do export PYTHON_VERSION=${pyver} @@ -24,11 +26,13 @@ do wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz tar -xzf Python-${PYTHON_VERSION}.tgz cd Python-${PYTHON_VERSION} - ./configure --prefix=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH} --enable-shared --enable-optimizations --enable-ipv6 LDFLAGS=-Wl,-rpath=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/lib,--disable-new-dtags + ./configure --prefix=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH} --enable-shared --enable-optimizations --enable-ipv6 --enable-loadable-sqlite-extensions LDFLAGS=-Wl,-rpath=/opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/lib,--disable-new-dtags make -j$(nproc) sudo make install sudo touch /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}.complete sudo ln -s /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/bin/python${PYTHON_MAJOR} /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/bin/python + sudo ln -s /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/bin/pip${PYTHON_MAJOR} /opt/runner/_work/_tool/Python/${PYTHON_VERSION}/${M_ARCH}/bin/pip cd .. rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz done + From 22e53de2125a1d9756093d56bc38c16c3e7efa69 Mon Sep 17 00:00:00 2001 From: Shahid Shaikh Date: Wed, 20 Mar 2024 15:31:38 -0400 Subject: [PATCH 5/6] Add multiple dotnet support --- Dockerfile.almalinux | 4 ++ Dockerfile.ubuntu | 5 ++- build-files/build-image.sh | 16 +++++++- build-files/install-dotnet.sh | 75 +++++++++++++++++++++++++++++++++++ setup-build-env.sh | 3 ++ 5 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 build-files/install-dotnet.sh diff --git a/Dockerfile.almalinux b/Dockerfile.almalinux index d90d307..38fe8cf 100644 --- a/Dockerfile.almalinux +++ b/Dockerfile.almalinux @@ -56,6 +56,10 @@ RUN if [ "${BTOOLS}" -eq 1 ]; then \ chown -R almalinux:almalinux /opt/runner; \ fi +RUN if [ "${BTOOLS}" -eq 1 ]; then \ + dnf install -y -q dotnet-sdk-[6-7-8]*.0; \ + fi + RUN rm -rf /tmp/runner /var/cache/dnf/* /tmp/runner.patch /tmp/install-python.sh /tmp/install-ruby.sh && \ dnf clean all diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index dffa13d..12b0502 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -51,6 +51,7 @@ RUN apt-get -qq -y install cmake make automake autoconf m4 gcc-12-base libto COPY build-files/install-python.sh /tmp COPY build-files/install-ruby.sh /tmp +COPY build-files/install-dotnet.sh /tmp RUN if [ "${BTOOLS}" -eq 1 ]; then \ apt-get -qq -y install openjdk-11-jdk libssl-dev ruby perl; \ @@ -58,10 +59,12 @@ RUN if [ "${BTOOLS}" -eq 1 ]; then \ /tmp/install-python.sh; \ chmod +x /tmp/install-ruby.sh; \ /tmp/install-ruby.sh; \ + chmod +x /tmp/install-dotnet.sh; \ + /tmp/install-dotnet.sh; \ chown -R ubuntu:ubuntu /opt/runner; \ fi -RUN rm -rf /tmp/runner /tmp/runner.patch /tmp/convert-rpm.sh /tmp/install-python.sh /tmp/install-ruby.sh +RUN rm -rf /tmp/runner /tmp/runner.patch /tmp/convert-rpm.sh /tmp/install-python.sh /tmp/install-ruby.sh /tmp/install-dotnet.sh USER ubuntu diff --git a/build-files/build-image.sh b/build-files/build-image.sh index a07c26c..20f0883 100755 --- a/build-files/build-image.sh +++ b/build-files/build-image.sh @@ -17,6 +17,11 @@ update_fresh_container() { } setup_dotnet_sdk() { + if [[ "${ARCH}" = "ppc64le" && ${SDK} -eq 6 ]]; then + echo "DOTNET ${SDK} is not supported on architecture ${ARCH}" >&2 + return 1 + fi + MIRROR="https://mirror.lchs.network/pub/almalinux/9.3/AppStream/${ARCH}/os/Packages" case "${SDK}" in 7) @@ -140,9 +145,17 @@ install_ruby() { return $? } +install_dotnet() { + echo "Installing DOTNET" + chmod +x /home/ubuntu/install-dotnet.sh + sudo /home/ubuntu/install-dotnet.sh + return $? +} + cleanup() { rm -rf /home/ubuntu/build-image.sh /home/ubuntu/runner-${ARCH}.patch \ - /tmp/runner /tmp/preseed-yaml + /tmp/runner /tmp/preseed-yaml \ + /home/ubuntu/install-python.sh /home/ubuntu/install-ruby.sh /home/ubuntu/install-dotnet.sh } run() { @@ -161,6 +174,7 @@ run() { if [ ${BTOOLS} -eq 1 ]; then install_python install_ruby + install_dotnet RC=$? fi fi diff --git a/build-files/install-dotnet.sh b/build-files/install-dotnet.sh new file mode 100644 index 0000000..6a59923 --- /dev/null +++ b/build-files/install-dotnet.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +retrieve_dotnet_pkgs() { + echo "Retrieving dotnet packages" + pushd /tmp >/dev/null + for pkg in ${PKGS} + do + RPM="${pkg}.${ARCH}.rpm" + wget -q ${MIRROR}/${RPM} + if [ ${ID} == "ubuntu" ]; then + echo -n "Converting ${RPM}... " + alien -d ${RPM} |& grep -v ^warning + if [ $? -ne 0 ]; then + exit 2 + fi + rm -f ${RPM} + fi + done + + echo "Installing dotnet packages" + dpkg --install /tmp/*.deb + rm -rf /tmp/*.deb +} + +. /etc/os-release +DOTNET_VERSIONS="6 7 8" +ARCH=`uname -m` +MIRROR="https://mirror.lchs.network/pub/almalinux/9.3/AppStream/${ARCH}/os/Packages" + +if [ "${ARCH}" == "ppc64le" ]; then + DOTNET_VERSIONS="8" +elif [ "${ARCH}" == "s390x" ]; then + DOTNET_VERSIONS="7 8" +else + DOTNET_VERSIONS="6 7 8" +fi + +for dotnetver in ${DOTNET_VERSIONS} +do +case "${dotnetver}" in + 8) + PKGS="dotnet-apphost-pack-8.0-8.0.1-1.el9_3 dotnet-host-8.0.1-1.el9_3" + PKGS="${PKGS} dotnet-hostfxr-8.0-8.0.1-1.el9_3 dotnet-targeting-pack-8.0-8.0.1-1.el9_3" + PKGS="${PKGS} dotnet-templates-8.0-8.0.101-1.el9_3 dotnet-runtime-8.0-8.0.1-1.el9_3" + PKGS="${PKGS} dotnet-sdk-8.0-8.0.101-1.el9_3 aspnetcore-runtime-8.0-8.0.1-1.el9_3" + PKGS="${PKGS} aspnetcore-targeting-pack-8.0-8.0.1-1.el9_3 netstandard-targeting-pack-2.1-8.0.101-1.el9_3" + retrieve_dotnet_pkgs + ;; + 7) + PKGS="dotnet-apphost-pack-7.0-7.0.15-1.el9_3 dotnet-host-8.0.1-1.el9_3" + PKGS="${PKGS} dotnet-hostfxr-7.0-7.0.15-1.el9_3 dotnet-targeting-pack-7.0-7.0.15-1.el9_3" + PKGS="${PKGS} dotnet-templates-7.0-7.0.115-1.el9_3 dotnet-runtime-7.0-7.0.15-1.el9_3" + PKGS="${PKGS} dotnet-sdk-7.0-7.0.115-1.el9_3 aspnetcore-runtime-7.0-7.0.15-1.el9_3" + PKGS="${PKGS} aspnetcore-targeting-pack-7.0-7.0.15-1.el9_3 netstandard-targeting-pack-2.1-8.0.101-1.el9_3" + retrieve_dotnet_pkgs + ;; + 6) + PKGS="dotnet-host-8.0.1-1.el9_3 dotnet-apphost-pack-6.0-6.0.26-1.el9_3" + PKGS="${PKGS} dotnet-hostfxr-6.0-6.0.26-1.el9_3 dotnet-targeting-pack-6.0-6.0.26-1.el9_3" + PKGS="${PKGS} dotnet-templates-6.0-6.0.126-1.el9_3 dotnet-runtime-6.0-6.0.26-1.el9_3" + PKGS="${PKGS} dotnet-sdk-6.0-6.0.126-1.el9_3 aspnetcore-runtime-6.0-6.0.26-1.el9_3" + PKGS="${PKGS} aspnetcore-targeting-pack-6.0-6.0.26-1.el9_3 netstandard-targeting-pack-2.1-8.0.101-1.el9_3" + retrieve_dotnet_pkgs + ;; + *) + echo "Unsupported DOTNET version ${dotnetver}" >&2 + exit 1 + ;; +esac +done + +echo "List all installed .NET SDKs - `dotnet --list-sdks`" + +exit 0 + diff --git a/setup-build-env.sh b/setup-build-env.sh index 2c7ad70..6db1c06 100755 --- a/setup-build-env.sh +++ b/setup-build-env.sh @@ -82,6 +82,9 @@ build_image_in_container() { echo "Copy the install-ruby script into gha-builder" lxc file push --mode 0755 "${BUILD_PREREQS_PATH}/install-ruby.sh" "${BUILD_CONTAINER}${BUILD_HOME}/install-ruby.sh" + echo "Copy the install-dotnet script into gha-builder" + lxc file push --mode 0755 "${BUILD_PREREQS_PATH}/install-dotnet.sh" "${BUILD_CONTAINER}${BUILD_HOME}/install-dotnet.sh" + echo "Running build-image.sh" lxc exec "${BUILD_CONTAINER}" --user 1000 --group 1000 -- ${BUILD_HOME}/build-image.sh -a ${ACTION_RUNNER} -s ${SDK} -t ${BTOOLS} RC=$? From 33be302556a4eb8fdf7286d3cdbbf7a7b4a1ebcd Mon Sep 17 00:00:00 2001 From: Shahid Shaikh Date: Thu, 21 Mar 2024 05:07:47 -0400 Subject: [PATCH 6/6] Incorporate review comments by Neale --- Dockerfile.almalinux | 8 +++----- Dockerfile.ubuntu | 9 +++------ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Dockerfile.almalinux b/Dockerfile.almalinux index 38fe8cf..6ea4a40 100644 --- a/Dockerfile.almalinux +++ b/Dockerfile.almalinux @@ -44,20 +44,18 @@ RUN mkdir -p /opt/runner && \ RUN dnf install -y -q cmake make automake autoconf m4 gcc gcc-c++ libtool epel-release -COPY build-files/install-python.sh /tmp -COPY build-files/install-ruby.sh /tmp +COPY --chmod=755 build-files/install-python.sh /tmp +COPY --chmod=755 build-files/install-ruby.sh /tmp RUN if [ "${BTOOLS}" -eq 1 ]; then \ dnf install -y -q java-11-openjdk openssl-devel ruby perl; \ - chmod +x /tmp/install-python.sh; \ /tmp/install-python.sh; \ - chmod +x /tmp/install-ruby.sh; \ /tmp/install-ruby.sh; \ chown -R almalinux:almalinux /opt/runner; \ fi RUN if [ "${BTOOLS}" -eq 1 ]; then \ - dnf install -y -q dotnet-sdk-[6-7-8]*.0; \ + dnf install -y -q dotnet-sdk-[6-7-8]*; \ fi RUN rm -rf /tmp/runner /var/cache/dnf/* /tmp/runner.patch /tmp/install-python.sh /tmp/install-ruby.sh && \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 12b0502..613d029 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -49,17 +49,14 @@ RUN mkdir -p /opt/runner && \ RUN apt-get -qq -y install cmake make automake autoconf m4 gcc-12-base libtool -COPY build-files/install-python.sh /tmp -COPY build-files/install-ruby.sh /tmp -COPY build-files/install-dotnet.sh /tmp +COPY --chmod=755 build-files/install-python.sh /tmp +COPY --chmod=755 build-files/install-ruby.sh /tmp +COPY --chmod=755 build-files/install-dotnet.sh /tmp RUN if [ "${BTOOLS}" -eq 1 ]; then \ apt-get -qq -y install openjdk-11-jdk libssl-dev ruby perl; \ - chmod +x /tmp/install-python.sh; \ /tmp/install-python.sh; \ - chmod +x /tmp/install-ruby.sh; \ /tmp/install-ruby.sh; \ - chmod +x /tmp/install-dotnet.sh; \ /tmp/install-dotnet.sh; \ chown -R ubuntu:ubuntu /opt/runner; \ fi