CustomPiOS v2 - suport for meta modules and remote modules

.github/workflows/docker-build.yml

@@ -6,6 +6,9 @@ on:
       - "master"
       - "devel"
       - "docker-github-actions"
+      - "feature/remote-module-support"
+      - "feature/meta-modules"
+      - "beta"
 
 jobs:
   docker:

README.rst

@@ -63,6 +63,7 @@ Requirements
 #. sudo (the script itself calls it, running as root without sudo won't work)
 #. p7zip-full
 #. Python 3.2+
+#. GitPython
 
 Known to work building configurations
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "custompios"
+version = "2.0.0"
+description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship."
+authors = ["Guy Sheffer <guysoft@gmail.com>"]
+license = "GPLv3"
+readme = "README.rst"
+packages = [
+#     { include = "src/*" },
+    { include = "custompios_core", from = "src" }
+    ]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+GitPython = "^3.1.41"
+
+[tool.poetry.group.dev.dependencies]
+types-PyYAML = "^6.0.12.12"
+
+[tool.poetry.scripts]
+custompios_build = 'custompios_core.multi_build:main'
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

src/Dockerfile

@@ -13,6 +13,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         wget \
         p7zip-full \
         python3 \
+        python3-distutils \
+        python3-dev \
+        python3-git \
+        python3-yaml \
         binfmt-support \
         qemu-system \
         qemu-user \

src/base_image_downloader_wrapper.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set +x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "$DIR/argparse.bash" || exit 1
+argparse "$@" <<EOF || exit 1
+
+parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+EOF
+
+if [ -z "${CUSTOM_PI_OS_PATH}" ];then
+    echo "Error: you must have \${CUSTOM_PI_OS_PATH} set"
+    exit 1
+fi
+
+# source "${DIST_PATH}/config"
+source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
+
+python3 ${CUSTOM_PI_OS_PATH}/custompios_core/base_image_downloader.py "${WORKSPACE_SUFFIX}"
+

src/build

@@ -10,15 +10,19 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
 
 define SCRIPT <<'EOF'
 BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
+export EXTRA_BOARD_CONFIG=$(mktemp)
+${BUILD_SCRIPT_PATH}/custompios_core/generate_board_config.py "${EXTRA_BOARD_CONFIG}"
+echo "Temp source file: ${EXTRA_BOARD_CONFIG}"
+
 source ${BUILD_SCRIPT_PATH}/common.sh
 install_cleanup_trap
 
 CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 
-source ${CUSTOM_PI_OS_PATH}/config ${@}
+source ${CUSTOM_PI_OS_PATH}/config "${1}" "${EXTRA_BOARD_CONFIG}" ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
-[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios
+[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}
 EOF
 
 if [ "$LOG" != "no" ]; then

src/common.sh

@@ -150,10 +150,13 @@ function unpack() {
 }
 
 function detach_all_loopback(){
+  image_name=$1
   # Cleans up mounted loopback devices from the image name
   # NOTE: it might need a better way to grep for the image name, its might clash with other builds
   for img in $(losetup  | grep $1 | awk '{ print $1 }' );  do
-    if [ -f "${img}" ] || [ -b "${img}" ]; then
+    # test if the image name is a substring
+    if [ "${img}" != "$(printf '%s' "${img}" | sed 's/'"${image_name}"'//g')" ] && ([ -f "${img}" ] || [ -b "${img}" ]); then
+      echo "freeing up $img"
     	losetup -d $img
     fi
   done
@@ -554,3 +557,36 @@ function set_config_var() {
   # See https://github.com/RPi-Distro/raspi-config/blob/master/raspi-config#L231
   raspi-config nonint set_config_var $1 $2 /boot/config.txt
 }
+
+
+function load_module_config() {
+  # Takes a comma seprated modules list, and exports the environment variables for it
+  MODULES_AFTER=$1
+  for module in $(echo "${MODULES_AFTER}" | tr "," "\n")
+  do
+      if [ -d "${DIST_PATH}/modules/${module}" ]; then
+          export MODULE_PATH="${DIST_PATH}/modules/${module}"
+      elif   [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
+          export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
+      fi
+
+      echo "loading $module config at ${MODULE_PATH}/config"
+      if [ -f "${MODULE_PATH}/config" ]; then
+          source "${MODULE_PATH}/config"
+      else
+          echo "WARNING: module ${module} has no config file"
+      fi
+
+      ###############################################################################
+      # Print and export the final configuration.
+
+      echo "================================================================"
+      echo "Using the following config:"
+      module_up=${module^^} module_up=${module_up//-/_}_
+
+      # Export variables that satisfy the $module_up prefix
+      while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
+
+      echo "================================================================"
+  done
+}

src/config

@@ -1,11 +1,14 @@
 CONFIG_DIR=$(dirname $(realpath -s "${BASH_SOURCE}"))
+source ${CUSTOM_PI_OS_PATH}/common.sh
 
 WORKSPACE_POSTFIX=
 
 export BUILD_VARIANT=""
 BUILD_VARIANT="$1"
 : ${BUILD_VARIANT:=default}
 
+EXTRA_BAORD_CONFIG=$2
+
 export BUILD_FLAVOR=""
 # Disable flavor system
 #BUILD_FLAVOR="$1"
@@ -79,31 +82,23 @@ TMP="${MODULES//(/,}"
 TMP="${TMP// /}"
 MODULES_LIST="${TMP//)/,}"
 
-for module in $(echo "${MODULES_LIST}" | tr "," "\n")
-do
-    if [ -d "${DIST_PATH}/modules/${module}" ]; then
-        export MODULE_PATH="${DIST_PATH}/modules/${module}"
-    elif   [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
-        export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
-    fi
-
-    echo "loading $module config at ${MODULE_PATH}/config"
-    if [ -f "${MODULE_PATH}/config" ]; then
-        source "${MODULE_PATH}/config"
-    else
-        echo "WARNING: module ${module} has no config file"
-    fi
-
-    ###############################################################################
-    # Print and export the final configuration.
-
-    echo "================================================================"
-    echo "Using the following config:"
-    module_up=${module^^} module_up=${module_up//-/_}_
-
-    # Export variables that satisfy the $module_up prefix
-    while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
-
-    echo "================================================================"
-done
 
+# Base workspace is special, it has to be sourced before the base module, so remote modules could be calcualted
+[ -n "$BASE_WORKSPACE" ] || BASE_WORKSPACE=${DIST_PATH}/workspace$WORKSPACE_POSTFIX
+# [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
+[ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
+
+# Import remote and submodules config
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+  source "${EXTRA_BAORD_CONFIG}"
+else
+  echo "Note: Not sourceing board config"
+fi
+
+export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
+# Remote modules and meta modulese go in first if they want to change standard behaviour
+if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+  source "${REMOTE_AND_META_CONFIG}"
+fi
+
+load_module_config "${MODULES_LIST}"

src/custompios

@@ -108,6 +108,11 @@ fi
 mkdir -p $BASE_WORKSPACE
 mkdir -p $BASE_MOUNT_PATH
 
+# This is already genrated at "build" sourced in "config", but copying here mostly for debug
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+  mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config
+fi
+
 # Clean exported artifacts from other builds
 rm -rf "${BASE_WORKSPACE}"/*.tar.gz
 
@@ -121,6 +126,7 @@ pushd $BASE_WORKSPACE
   fi
   if [ ! -f "$BASE_ZIP_IMG" ] || [ "$BASE_ZIP_IMG" == "" ]; then
     echo "Error: could not find image: $BASE_ZIP_IMG"
+    echo "On CustomPiOS v2 you can provide -d to download the latest image of your board automatically"
     exit 1
   fi
 
@@ -179,7 +185,20 @@ pushd $BASE_WORKSPACE
     # execute the base chroot script
     ### execute_chroot_script $BASE_SCRIPT_PATH $BASE_CHROOT_SCRIPT_PATH
     CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
-    python3 ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" ${CHROOT_SCRIPT}
+    MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
+    MODULES_BEFORE="${MODULES}"
+    ${CUSTOM_PI_OS_PATH}/custompios_core/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
+    if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+      echo "Sourcing remote and submodules config"
+      source "${REMOTE_AND_META_CONFIG}" ${@}
+
+      MODULES_AFTER=$(cat "${MODULES_AFTER_PATH}")
+      load_module_config "${MODULES_AFTER}"
+
+    else
+      echo "No remote and submodules config detected"
+    fi
+    echo $ARMBIAN_CONFIG_TXT_FILE
     export -f execute_chroot_script
     bash -x "${CHROOT_SCRIPT}"