From cf8a0e182bbc090af1d4905c11526dfa876b7e78 Mon Sep 17 00:00:00 2001 From: Keli Grubb Date: Thu, 23 Jan 2020 10:52:57 -0500 Subject: [PATCH] Consolidate build scripts (#286) * make sure all scripts exit on failure. * move all terraform script tooling into the build script. Add a new option to allow config to choose whether or not to include the appcenter ppa. * clean up comments related to old terraform script * remove artifacts dir step, direct output to builds/amd64. * move comments to better locations for logging * allow for uploading other arch types in the future * remove builds dir before moving over new builds * Remove the builds directory before all build runs instead of before each. --- .github/workflows/daily-5.1.yml | 1 - .github/workflows/daily-6.0.yml | 1 - .github/workflows/distinst-weekly.yml | 1 - .github/workflows/stable.yml | 1 - .gitignore | 1 + README.md | 7 +- build.sh | 87 ++++++++++++-- .../appcenter.key.binary | 0 .../appcenter.list.binary | 0 etc/terraform-daily-5.1-azure.conf | 3 + etc/terraform-daily-6.0-azure.conf | 3 + etc/terraform-daily-distinst-azure.conf | 3 + etc/terraform-stable-azure.conf | 3 + etc/terraform.conf | 3 + terraform.sh | 108 ------------------ upload.sh | 27 +++-- workflows.sh | 2 + 17 files changed, 115 insertions(+), 136 deletions(-) rename etc/config/{archives => appcenter}/appcenter.key.binary (100%) rename etc/config/{archives => appcenter}/appcenter.list.binary (100%) delete mode 100755 terraform.sh diff --git a/.github/workflows/daily-5.1.yml b/.github/workflows/daily-5.1.yml index ce3316f56..26d2dece9 100644 --- a/.github/workflows/daily-5.1.yml +++ b/.github/workflows/daily-5.1.yml @@ -22,5 +22,4 @@ jobs: - name: Build and upload daily .iso run: | - mkdir /artifacts ./workflows.sh etc/terraform-daily-5.1-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}" diff --git a/.github/workflows/daily-6.0.yml b/.github/workflows/daily-6.0.yml index 4d0d22685..a0ac5f1ff 100644 --- a/.github/workflows/daily-6.0.yml +++ b/.github/workflows/daily-6.0.yml @@ -22,5 +22,4 @@ jobs: - name: Build and upload daily .iso run: | - mkdir /artifacts ./workflows.sh etc/terraform-daily-6.0-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}" diff --git a/.github/workflows/distinst-weekly.yml b/.github/workflows/distinst-weekly.yml index 35c35cc9d..749e72d30 100644 --- a/.github/workflows/distinst-weekly.yml +++ b/.github/workflows/distinst-weekly.yml @@ -19,5 +19,4 @@ jobs: - name: Build and upload distinst .iso run: | - mkdir /artifacts ./workflows.sh etc/terraform-daily-distinst-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}" diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index c634d74cc..69a0a20a6 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -20,5 +20,4 @@ jobs: - name: Build and upload stable .iso run: | - mkdir /artifacts ./workflows.sh etc/terraform-stable-azure.conf "${{ secrets.key }}" "${{ secrets.secret }}" "${{ secrets.endpoint }}" "${{ secrets.bucket }}" diff --git a/.gitignore b/.gitignore index 7e4f60d4e..6bfb040f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store tmp artifacts +builds *~ diff --git a/README.md b/README.md index 402cbdc91..69292c213 100644 --- a/README.md +++ b/README.md @@ -36,17 +36,14 @@ The following example uses Docker and assumes you have Docker correctly installe 3) Run the build: ``` - mkdir artifacts - docker run --privileged -i \ - -v /proc:/proc \ - -v ${PWD}/artifacts:/artifacts \ + docker run --privileged -i -v /proc:/proc \ -v ${PWD}:/working_dir \ -w /working_dir \ debian:latest \ /bin/bash -s etc/terraform.conf < build.sh ``` - 4) When done, your image will be in the `artifacts` folder. + 4) When done, your image will be in the `builds` folder. ## Further Information diff --git a/build.sh b/build.sh index 285b4f5aa..a25d66037 100755 --- a/build.sh +++ b/build.sh @@ -1,8 +1,21 @@ #!/bin/bash -CONFIG_FILE="$1" +set -e -source "$CONFIG_FILE" +# check for root permissions +if [[ "$(id -u)" != 0 ]]; then + echo "E: Requires root permissions" > /dev/stderr + exit 1 +fi + +# get config +if [ -n "$1" ]; then + CONFIG_FILE="$1" +else + CONFIG_FILE="etc/terraform.conf" +fi +BASE_DIR="$PWD" +source "$BASE_DIR"/"$CONFIG_FILE" echo -e " #----------------------# @@ -21,11 +34,69 @@ patch -d /usr/lib/live/build/ < live-build-fix-syslinux.patch # https://salsa.debian.org/installer-team/debootstrap/blob/master/debian/changelog ln -sfn /usr/share/debootstrap/scripts/gutsy /usr/share/debootstrap/scripts/focal -echo -e " -#----------------------# -# RUN TERRAFORM SCRIPT # -#----------------------# +build () { + BUILD_ARCH="$1" + + mkdir -p "$BASE_DIR/tmp/$BUILD_ARCH" + cd "$BASE_DIR/tmp/$BUILD_ARCH" || exit + + # remove old configs and copy over new + rm -rf config auto + cp -r "$BASE_DIR"/etc/* . + # Make sure conffile specified as arg has correct name + cp -f "$BASE_DIR"/"$CONFIG_FILE" terraform.conf + + # Symlink chosen package lists to where live-build will find them + ln -s "package-lists.$PACKAGE_LISTS_SUFFIX" "config/package-lists" + # symlink appcenter archive + if [ "$INCLUDE_APPCENTER" = "yes" ]; then + ln -s "appcenter/appcenter.list.binary" "archives/appcenter.list.binary" + ln -s "appcenter/appcenter.key.binary" "archives/appcenter.key.binary" + fi + + echo -e " +#------------------# +# LIVE-BUILD CLEAN # +#------------------# " + lb clean + + echo -e " +#-------------------# +# LIVE-BUILD CONFIG # +#-------------------# +" + lb config + + echo -e " +#------------------# +# LIVE-BUILD BUILD # +#------------------# +" + lb build + + echo -e " +#---------------------------# +# MOVE OUTPUT TO BUILDS DIR # +#---------------------------# +" + + YYYYMMDD="$(date +%Y%m%d)" + OUTPUT_DIR="$BASE_DIR/builds/$BUILD_ARCH" + mkdir -p "$OUTPUT_DIR" + FNAME="elementaryos-$VERSION-$CHANNEL.$YYYYMMDD$OUTPUT_SUFFIX" + mv "$BASE_DIR/tmp/$BUILD_ARCH/live-image-$BUILD_ARCH.hybrid.iso" "$OUTPUT_DIR/${FNAME}.iso" + + md5sum "$OUTPUT_DIR/${FNAME}.iso" > "$OUTPUT_DIR/${FNAME}.md5.txt" + sha256sum "$OUTPUT_DIR/${FNAME}.iso" > "$OUTPUT_DIR/${FNAME}.sha256.txt" +} + +# remove old builds before creating new ones +rm -rf "$BASE_DIR"/builds -./terraform.sh --config-path "$CONFIG_FILE" -cp builds/amd64/* /artifacts/ +if [[ "$ARCH" == "all" ]]; then + build amd64 + build i386 +else + build "$ARCH" +fi diff --git a/etc/config/archives/appcenter.key.binary b/etc/config/appcenter/appcenter.key.binary similarity index 100% rename from etc/config/archives/appcenter.key.binary rename to etc/config/appcenter/appcenter.key.binary diff --git a/etc/config/archives/appcenter.list.binary b/etc/config/appcenter/appcenter.list.binary similarity index 100% rename from etc/config/archives/appcenter.list.binary rename to etc/config/appcenter/appcenter.list.binary diff --git a/etc/terraform-daily-5.1-azure.conf b/etc/terraform-daily-5.1-azure.conf index 66b1b8f99..e60ee7879 100644 --- a/etc/terraform-daily-5.1-azure.conf +++ b/etc/terraform-daily-5.1-azure.conf @@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/" # use HWE kernel and packages? HWE="yes" +# use appcenter ppa +INCLUDE_APPCENTER="yes" + # suffix for generated .iso files OUTPUT_SUFFIX="" diff --git a/etc/terraform-daily-6.0-azure.conf b/etc/terraform-daily-6.0-azure.conf index 2391e7d96..20a8e4165 100644 --- a/etc/terraform-daily-6.0-azure.conf +++ b/etc/terraform-daily-6.0-azure.conf @@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/" # use HWE kernel and packages? HWE="" +# use appcenter ppa +INCLUDE_APPCENTER="" + # suffix for generated .iso files OUTPUT_SUFFIX="-distinst" diff --git a/etc/terraform-daily-distinst-azure.conf b/etc/terraform-daily-distinst-azure.conf index a25f1c888..16a5eb136 100644 --- a/etc/terraform-daily-distinst-azure.conf +++ b/etc/terraform-daily-distinst-azure.conf @@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/" # use HWE kernel and packages? HWE="yes" +# use appcenter ppa +INCLUDE_APPCENTER="yes" + # suffix for generated .iso files OUTPUT_SUFFIX="-distinst" diff --git a/etc/terraform-stable-azure.conf b/etc/terraform-stable-azure.conf index 6687e3a10..4af4331e1 100644 --- a/etc/terraform-stable-azure.conf +++ b/etc/terraform-stable-azure.conf @@ -25,6 +25,9 @@ MIRROR_URL="http://azure.archive.ubuntu.com/ubuntu/" # use HWE kernel and packages? HWE="yes" +# use appcenter ppa +INCLUDE_APPCENTER="yes" + # suffix for generated .iso files OUTPUT_SUFFIX="" diff --git a/etc/terraform.conf b/etc/terraform.conf index a2309c735..4908b9657 100644 --- a/etc/terraform.conf +++ b/etc/terraform.conf @@ -25,6 +25,9 @@ MIRROR_URL="http://archive.ubuntu.com/ubuntu/" # use HWE kernel and packages? HWE="yes" +# use appcenter ppa +INCLUDE_APPCENTER="yes" + # suffix for generated .iso files OUTPUT_SUFFIX="" diff --git a/terraform.sh b/terraform.sh deleted file mode 100755 index 80e8a2ea8..000000000 --- a/terraform.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -set -e - -CONFIG_PATH="etc/terraform.conf" - -# Call getopt to validate the provided input. -options=$(getopt -o '' -l 'config-path:' -- "$@") -[ $? -eq 0 ] || { - echo "Incorrect options provided" - exit 1 -} -eval set -- "$options" -while true; do - case "$1" in - --config-path) - shift; # The arg is next in position args - CONFIG_PATH=$1 - ;; - --) - shift - break - ;; - esac - shift -done - -check_permissions () { - if [[ "$(id -u)" != 0 ]]; then - echo "E: Requires root permissions" > /dev/stderr - exit 1 - fi -} - - -check_dependencies () { - PACKAGES="live-build" - for PACKAGE in $PACKAGES; do - dpkg -L "$PACKAGE" >/dev/null 2>&1 || MISSING_PACKAGES="$MISSING_PACKAGES $PACKAGE" - done - - if [[ "$MISSING_PACKAGES" != "" ]]; then - echo "E: Missing dependencies! Please install the following packages: $MISSING_PACKAGES" > /dev/stderr - exit 1 - fi -} - -read_config () { - BASE_DIR="$PWD" - source "$BASE_DIR"/"$CONFIG_PATH" -} - -build () { - BUILD_ARCH="$1" - - mkdir -p "$BASE_DIR/tmp/$BUILD_ARCH" - cd "$BASE_DIR/tmp/$BUILD_ARCH" || exit - - # remove old configs and copy over new - rm -rf config auto - cp -r "$BASE_DIR"/etc/* . - # Make sure conffile specified as arg has correct name - cp -f "$BASE_DIR"/"$CONFIG_PATH" terraform.conf - - # Symlink chosen package lists to where live-build will find them - ln -s "package-lists.$PACKAGE_LISTS_SUFFIX" "config/package-lists" - - echo -e " -#------------------# -# LIVE-BUILD CLEAN # -#------------------# -" - lb clean - - echo -e " -#-------------------# -# LIVE-BUILD CONFIG # -#-------------------# -" - lb config - - echo -e " -#------------------# -# LIVE-BUILD BUILD # -#------------------# -" - lb build - - YYYYMMDD="$(date +%Y%m%d)" - OUTPUT_DIR="$BASE_DIR/builds/$BUILD_ARCH" - mkdir -p "$OUTPUT_DIR" - FNAME="elementaryos-$VERSION-$CHANNEL.$YYYYMMDD$OUTPUT_SUFFIX" - mv "$BASE_DIR/tmp/$BUILD_ARCH/live-image-$BUILD_ARCH.hybrid.iso" "$OUTPUT_DIR/${FNAME}.iso" - - md5sum "$OUTPUT_DIR/${FNAME}.iso" > "$OUTPUT_DIR/${FNAME}.md5.txt" - sha256sum "$OUTPUT_DIR/${FNAME}.iso" > "$OUTPUT_DIR/${FNAME}.sha256.txt" -} - -check_permissions -check_dependencies -read_config - -if [[ "$ARCH" == "all" ]]; then - build amd64 - build i386 -else - build "$ARCH" -fi diff --git a/upload.sh b/upload.sh index 096cb64c0..9b0c2214a 100755 --- a/upload.sh +++ b/upload.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + CONFIG_FILE="$1" KEY="$2" SECRET="$3" @@ -28,14 +30,17 @@ python3 get-pip.py pip install boto3 # get the paths & filenames of the files to upload -ISOPATH="$(find /artifacts -name "*.iso")" -ISO="$CHANNEL/$(basename "$ISOPATH")" -ISOTAG="$(basename "$ISOPATH" .iso)" -SHAPATH="$(find /artifacts -name "*.sha256.txt")" -SHASUM="$CHANNEL/$ISOTAG.sha256.txt" -MD5PATH="$(find /artifacts -name "*.md5.txt")" -MD5="$CHANNEL/$ISOTAG.md5.txt" - -python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$ISOPATH" "$ISO" -python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$SHAPATH" "$SHASUM" -python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$MD5PATH" "$MD5" +ISOPATHS="$(find builds -name "*.iso")" +while IFS= read -r ISOPATH; do + SHAPATH="${ISOPATH%.*}.sha256.txt" + MD5PATH="${ISOPATH%.*}.md5.txt" + ISO="$CHANNEL/$(basename "$ISOPATH")" + SHASUM="$CHANNEL/$(basename "$SHAPATH")" + MD5="$CHANNEL/$(basename "$MD5PATH")" + echo "uploading $ISO..." + python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$ISOPATH" "$ISO" || exit 1 + echo "uploading $SHASUM..." + python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$SHAPATH" "$SHASUM" || exit 1 + echo "uploading $MD5..." + python3 upload.py "$KEY" "$SECRET" "$ENDPOINT" "$BUCKET" "$MD5PATH" "$MD5" || exit 1 +done <<< "$ISOPATHS" diff --git a/workflows.sh b/workflows.sh index 229f02705..2dc80485e 100755 --- a/workflows.sh +++ b/workflows.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + CONFIG_FILE="$1" KEY="$2" SECRET="$3"