build_fs

#!/bin/sh
#  shellcheck disable=SC2154
#
#  Part of https://github.com/jaclu/AOK-Filesystem-Tools
#
#  Copyright (c) 2021-2024: Jacob.Lundqvist@gmail.com
#
#  License: MIT
#
#  Creates a FS image suitable for mounting on iSH, the build should
#  be able to complete on any platform.
#  Some options needs chrooting, those steps can only be done when
#  running iSH or Linux(x86)
#

show_help() { # Multi OK 1
    #region help text
    echo "Usage: $prog_name [-h] [-v] [-p] [-c] [-z] [-u]

This builds the iSH-AOK filesystem.

Default is to setup a minimal FS to be completed when deployed, this creates
far smaller image files, at the cost of taking a couple of minutes to complete
on the device upon first boot.

Available options:

-h  --help         Print this help and exit
-v  --verbose      Displays extra info, like untaring/taring progress
-s  --select       Offer selection between Alpine, Debian & Devuan on first boot.
-d  --debian       Build a Debian FS.
-D  --devuan       Build a Devuan FS.
-p  --prebuilt     Build the entire FS on the running platform, making
                   for quicker imports on target device, but up to 10
                   times larger image file.
                   This can only be done on iSH or Linux (x86)!
-f  --force        Force do_chroot.sh to run, despite any warnings
                   it displays.
-c  --clear        Empty the build env ($d_build_root)
-C  --full_clear   Do a clear, and in addition clear the download cache
-j  --bzip2        Use bzip2 compression for distribution
-N  --no_compress  Terminates when FS is prepared, without generating
                   a FS tarball.

A newly created FS is located at: $d_build_root

If there are remining deploy steps, they will be auto-performed
when you chroot or first boot into it using iSH.

If you want to access the FS without risking to move the deploy
forward, then you can do it like this: ./tools/do_chroot.sh /bin/bash

For normal chroot, just do:  ./tools/do_chroot.sh

After potential FS modifications, you can create a tarball from the FS
by running:  ./compress_image

Then just import this into iSH!
"
    #endregion
    exit 0
}

check_dependencies() {
    _deps=""
    [ -z "$(command -v rsync)" ] && _deps="$_deps rsync"
    _wget_cmd="$(command -v wget)"
    if [ -z "$_wget_cmd" ]; then
        _deps="$_deps wget"
    elif [ -n "$_wget_cmd" ] && readlink "$_wget_cmd" | grep -q busybox; then
        _deps="$_deps wget"
        msg_3 "You need to install a real wget, the busybox one does not handle redirects"
    fi
    [ -z "$_deps" ] && return

    msg_1 "In order for this to work some dependencies will be installed"

    case "$(hostfs_detect)" in

    "$distro_alpine")
        #  shellcheck disable=SC2086 # in this case it should expand
        apk add $_deps
        ;;
    "$distro_debian" | "$distro_devuan")
        #  shellcheck disable=SC2086 # in this case it should expand
        apt install $_deps
        ;;
    *)
        echo "Platform not recognized, can not suggest how to install"
        error_msg "You need the following packages: $_deps"
        ;;
    esac

    msg_3 "Dependencies resolved"
}

parse_arguments() { # Multi OK 1
    while [ -n "$1" ]; do

        case "$1" in

        "-h" | "--help") show_help ;;

        "-v" | "--verbose") verbose=true ;;

        "-d" | "--debian")
            if [ "$build_target" = "$destfs_select" ]; then
                echo "Can not be combined: -d and -s"
                exit 1
            fi
            build_target="$distro_debian"
            ;;

        "-D" | "--devuan")
            if [ "$build_target" = "$destfs_select" ]; then
                echo "Can not be combined: -D and -s"
                exit 1
            fi
            build_target="$distro_devuan"
            ;;

        "-s" | "--select")
            if [ "$build_target" = "$distro_debian" ] ||
                [ "$build_target" = "$distro_devuan" ]; then
                echo "Can not be combined: -d/-D and -s"
                exit 1
            fi
            if $pre_build_FS; then
                echo "Can not be combined: -p and -s"
                exit 1
            fi
            build_target="$destfs_select"
            ;;

        "-p" | "--prebuilt")
            if [ "$build_target" = "$destfs_select" ]; then
                echo "Can not be combined: -p and -s"
                exit 1
            fi
            pre_build_FS=true
            ;;

        "-f" | "--force")
            force_chroot="-f"
            ;;

        "-c" | "--clear")
            do_clear
            exit 0
            ;;

        "-C" | "--full_clear")
            do_full_clear
            exit 0
            ;;

        "-j" | "--bzip2")
            if ! $aok_FS_do_compress; then
                echo "Can not be combined: -j and -N"
                exit 1
            fi
            use_bzip2=true
            ;;

        "-N" | "--no_compress")
            if $use_bzip2; then
                echo "Can not be combined: -j and -N"
                exit 1
            fi
            aok_FS_do_compress=false
            ;;

        *)
            echo "ERROR: bad param, try -h"
            exit 1
            ;;

        esac
        shift
    done
}

do_clear() {

    # msg_2 "do_clear()"

    [ ! -e "$d_build_root" ] && error_msg "Buildenv not pressent: $d_build_root"
    rm -rf "$d_build_root"
    [ -e "$d_build_root" ] && error_msg "Failed to clear: $d_build_root"
    # msg_3 "do_clear() - done"
}

do_full_clear() {
    # msg_2 "do_full_clear()"
    do_clear
    [ -e "$d_src_img_cache" ] && rm -rf "$d_src_img_cache"
    [ -e "$d_src_img_cache" ] && error_msg "Failed to clear: $d_src_img_cache"
    # msg_3 "do_full_clear() - done"
}

display_build_target() { # Multi OK 1
    #
    #  Displaying build environment
    #
    [ "$build_env" = "$be_ish" ] && dbt_run_mode="Building on iSH"
    [ "$build_env" = "$be_linux" ] && dbt_run_mode="Building on x86 Linux"

    if ! $pre_build_FS; then
        dbt_run_mode="Will not chroot"
    elif [ "$build_env" = "$be_other" ]; then
        echo "Unfortunately, you can not chroot into the image on this device"
        echo "This is only supported on iSH and Linux(x86)"
        echo "Use another build option (try -h for help)"
        exit 1
    fi
    msg_1 "Run mode: $dbt_run_mode"
    unset dbt_run_mode

    if [ "$(whoami)" != "root" ]; then
        # Must come after help display, to avoid infinite loop
        "$0" -h
        error_msg "This must be run as root or using sudo!"
    fi

    if [ -d "$d_build_root" ] && [ -f "$d_build_root/$f_host_fs_is_chrooted" ]; then
        echo "ERROR: Active chroot session detected at $d_build_root!"
        echo "       If this is due to a crash or abort, you can clear it by running:"
        echo "         tools/do_chroot.sh -c"
        echo
        exit 1
    fi

    if [ "$build_target" = "$distro_debian" ]; then
        release_msg="Debian"
    elif [ "$build_target" = "$distro_devuan" ]; then
        release_msg="Devuan"
    else
        [ -z "$ALPINE_VERSION" ] && error_msg "ALPINE_VERSION is undefined"
        release_msg="Alpine: $ALPINE_VERSION"
    fi
    msg_1 "Building iSH-AOK $release_msg filesystem - $AOK_VERSION"
    unset release_msg

    if ! $aok_FS_do_compress; then
        msg_2 "*** Will not create the compressed image! ***"
    fi
    echo
}

cache_fs_image() {
    msg_2 "cache_fs_image()"
    [ -z "$src_image" ] && error_msg "cache_fs_image() no src_image supplied"
    [ -z "$src_tarball" ] && error_msg "cache_fs_image() no src_taball supplied"

    if [ ! -d "$d_src_img_cache" ]; then
        mkdir -p "$d_src_img_cache"
    fi
    cd "$d_src_img_cache" || {
        error_msg "Failed to cd to $d_src_img_cache"
    }
    if [ ! -f "$d_src_img_cache/$src_tarball" ]; then
        cmd="wget --no-check-certificate $src_image"
        #  Ensure basename for tar ball is used
        case "$build_target" in
        "$distro_alpine" | "$destfs_select") cmd="$cmd -O $alpine_src_tb" ;;
        "$distro_debian") cmd="$cmd -O $debian_src_tb" ;;
        "$distro_devuan") cmd="$cmd -O $devuan_src_tb" ;;
        *)
            error_msg "Invalid build_target: $build_target"
            ;;
        esac
        $cmd || {
            echo "ERROR: Failed to download $src_image"
            exit 1
        }
    fi
    # msg_3 "cache_fs_image() done"
}

copy_AOK_to_dest() {
    #
    #  Copy AOK content to destination
    #
    msg_2 "copy_AOK_to_dest()"

    mkdir -p "$aok_files"
    mkdir -p "$d_build_root"/etc/opt
    rsync_chown /opt/AOK/ "$d_build_root"/opt/AOK silent
    # msg_3 "copy_AOK_to_dest() done"
}

setup_Alpine_on_1st_boot() {
    msg_2 "setup_Alpine_on_1st_boot()"

    # buildtype_set "$distro_alpine"
    set_new_etc_profile "$setup_alpine_scr"
    # msg_3 "setup_Alpine_on_1st_boot() done"
}

setup_Debian_on_1st_boot() {
    msg_2 "setup_Debian_on_1st_boot()"

    # shellcheck source=/opt/AOK/FamDeb/deb_utils.sh
    . /opt/AOK/FamDeb/deb_utils.sh
    initial_fs_prep_fam_deb

    # buildtype_set "$distro_debian"
    set_new_etc_profile "$setup_debian_scr"

    # msg_3 "setup_Debian_on_1st_boot() done"
}

setup_Devuan_on_1st_boot() {
    msg_2 "setup_Devuan_on_1st_boot()"

    # shellcheck source=/opt/AOK/FamDeb/deb_utils.sh
    . /opt/AOK/FamDeb/deb_utils.sh
    initial_fs_prep_fam_deb

    # buildtype_set "$distro_devuan"
    set_new_etc_profile "$setup_devuan_scr"

    # msg_3 "setup_Devuan_on_1st_boot() done"
}

select_distro_on_1st_boot() {
    msg_2 "select_distro_on_1st_boot()"

    # buildtype_set "$destfs_select"
    touch "$f_destfs_select_hint"

    set_new_etc_profile "$setup_select_distro_prepare"

    #
    #  If this build platform cant chroot, then the preparational steps
    #  will be run on 1st boot, a slight delay, but end result is
    #  the same.
    #
    if [ "$build_env" != "$be_other" ]; then
        [ -n "$force_chroot" ] && msg_3 "Forcing chroot to run"
        if ! /opt/AOK/tools/do_chroot.sh "$force_chroot" /etc/profile; then
            error_msg "Error in chroot, aborting build!"
        fi
    else
        echo
        msg_2 "***  Could not chroot, did not run: $setup_select_distro_prepare  ***"
        msg_2 "This means that this image will have to run the preparatory"
        msg_2 "step, before displaying the Distro selection dialog"
        echo
    fi
    # msg_3 "select_distro_on_1st_boot()  done"
}

prebuild_fs() {
    msg_1 "prebuild_fs()"

    if [ "$build_env" = "$be_other" ]; then
        error_msg "Not possible to pre-build on this environment!"
    fi
    #
    # chrooting and doing setup
    #
    cd /opt/AOK || {
        error_msg "Failed to cd into: /opt/AOK"
    }

    deploy_state_set "$deploy_state_pre_build"

    [ -n "$force_chroot" ] && msg_3 "Forcing chroot to run"

    /opt/AOK/tools/do_chroot.sh "$force_chroot" /etc/profile || {
        error_msg "There was an error during prebuild"
    }

    msg_1 "Returned from prebuild chroot"

    #
    #  This ensures openrc doesnt detect previous runstates during
    #  1st boot and thus triggering some errors during boot & sysinit
    #
    msg_2 "Clearing run states after pre-build"
    rm "$d_build_root"/run/openrc -rf

    #  Make sure no leftover fixdev pid file remains
    rm "$d_build_root"/tmp/fixdev.pid -f

    echo
    # msg_3 "prebuild_fs() done"
}

consider_generating_tarball() {
    msg_2 "consider_generating_tarball()"
    #
    #  Consider to generate a tarball
    #
    if $aok_FS_do_compress; then
        cgt_extra_params=""
        if $verbose; then
            cgt_extra_params="-v"
        fi
        if $use_bzip2; then
            cgt_extra_params="$cgt_extra_params -j"
        fi

        if [ -n "$d_build_root" ] && ls -A "$d_build_root"/iCloud/* >/dev/null 2>&1; then
            #
            #  iCloud might have been used whilst chrooted, this ensures
            #  that it is empty when FS is compressed
            #
            msg_3 "Clearing /iCloud before generating image"
            rm "$d_build_root"/iCloud/* -rf
        fi

        #
        #  Here the tarball is generated
        #
        #  shellcheck disable=SC2086,SC2248
        if ! /opt/AOK/compress_image $cgt_extra_params; then
            error_msg "detected in compress_image"
        fi

        unset cgt_extra_params
        unset cgt_label
    fi

    # msg_3 "consider_generating_tarball() done"
}

display_build_time() {
    msg_2 "display_build_time()"
    dbt_duration="$(($(date +%s) - t_build_start))"
    if $pre_build_FS; then
        dbt_lbl="Create and setup FS"
    else
        dbt_lbl="Create FS"
    fi
    display_time_elapsed "$dbt_duration" "$dbt_lbl"
    unset dbt_duration
    unset dbt_lbl
    # msg_3 "display_build_time() done"
}

#===============================================================
#
#   Main
#
#===============================================================

hide_run_as_root=1 . /opt/AOK/tools/run_as_root.sh
[ -z "$d_aok_etc" ] && . /opt/AOK/tools/utils.sh

t_build_start="$(date +%s)"

# shellcheck disable=SC1007
prog_name=$(basename "$0")

deploy_state_set "$deploy_state_na" # Dest FS not yet created

#
#  Point to AOK-Filesystem-Tools dir, in case this was run with a path
#  to keep things relative and simple
#
#  shellcheck disable=SC1007
cd /opt/AOK || {
    error_msg "Failed to cd into: /opt/AOK"
}

#
#  Default for options
#
build_target="$distro_alpine" # default target
verbose=false                 # true displays tar/untar progress
use_bzip2=false               # Use tgz when compressing image
pre_build_FS=false            # true means generate pre-built FS
aok_FS_do_compress=true       # compress FS at end of build_fs

aok_files="${d_build_root}"/opt/AOK

parse_arguments "$@"

check_dependencies

this_fs_is_chrooted && error_msg "You can't build an FS when already chrooted!"

display_build_target

msg_2 "Preparing build environment"

[ -z "$d_build_root" ] && error_msg "d_build_root not assigned"
[ -d "$d_build_root" ] || error_msg "d_build_root not a folder [$d_build_root]"

msg_3 "Clearing build root $d_build_root"
rm -rf "$d_build_root"

if [ "$build_target" = "$distro_debian" ]; then
    src_image="$DEBIAN_SRC_IMAGE"
    src_tarball="$debian_src_tb"
elif [ "$build_target" = "$distro_devuan" ]; then
    src_image="$DEVUAN_SRC_IMAGE"
    src_tarball="$devuan_src_tb"
else
    src_image="$alpine_src_image"
    src_tarball="$alpine_src_tb"
fi

cache_fs_image
create_fs "${d_src_img_cache}/$src_tarball" "$d_build_root" "$verbose"

copy_AOK_to_dest
msg_3 "Deployed /opt/AOK on new filesystem"

#  Announce what AOK release this is
msg_2 "Set $(echo "$f_aok_fs_release" | sed "s|$d_build_root||") to $AOK_VERSION"

echo "$AOK_VERSION" >"$f_aok_fs_release"

#
#  Alpine initial motd suggests running setup-alpine. Not available on iSH
#  Since setup_alpine.sh will create an AOK relevant motd,
#  deleting the default one both on Alpine & Debian makes sence.
#
msg_2 "Remove initial /etc/motd from new filesystem"
rm "$d_build_root"/etc/motd

# Should always happen
msg_2 "Create directory /iCloud inside new filesystem"
mkdir -p "$d_build_root"/iCloud

if [ -n "$POPULATE_FS" ]; then
    #
    #  So far unofficial hack, if you put d_build_root in POPULATE_FS
    #  it will be replaced with actual build_root during creation of
    #  FS, most likely will never be used by anyone but me during chroot
    #  test deploys
    #
    pop_fs="$(echo "$POPULATE_FS" | sed "s#D_BUILD_ROOT#$d_build_root#g")"
    msg_2 "POPULATE_FS requested"
    echo "$pop_fs"
    echo
    /bin/sh -c "$pop_fs" || {
        error_msg "POPULATE_FS [$pop_fs] returned error"
    }
fi

if [ "$build_target" = "$distro_debian" ]; then
    setup_Debian_on_1st_boot
elif [ "$build_target" = "$distro_devuan" ]; then
    setup_Devuan_on_1st_boot
elif [ "$build_target" = "$destfs_select" ]; then
    select_distro_on_1st_boot
else
    setup_Alpine_on_1st_boot
fi

if $pre_build_FS; then
    prebuild_fs
fi

msg_2 "The filesystem is ready!"

consider_generating_tarball

display_build_time