Major rework for CustomPiOs v2 - add config loading of meta and remot…

…e modules, get meta modules and images name to match #214

src/base_image_downloader.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+import argparse
+import yaml
+import os
+import urllib.request
+import tempfile
+import hashlib
+import shutil
+import re
+PRECENT_PROGRESS_SIZE = 5
+
+class ChecksumFailException(Exception):
+ pass
+
+IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
+RETRY = 3
+
+def ensure_dir(d, chmod=0o777):
+ """
+ Ensures a folder exists.
+ Returns True if the folder already exists
+ """
+ if not os.path.exists(d):
+ os.makedirs(d, chmod)
+ os.chmod(d, chmod)
+ return False
+ return True
+
+def read_images():
+ if not os.path.isfile(IMAGES_CONFIG):
+ raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
+ with open(IMAGES_CONFIG,'r') as f:
+ output = yaml.safe_load(f)
+ return output
+
+class DownloadProgress:
+ last_precent: float = 0
+ def show_progress(self, block_num, block_size, total_size):
+ new_precent = round(block_num * block_size / total_size * 100, 1)
+ if new_precent > self.last_precent + PRECENT_PROGRESS_SIZE:
+ print(f"{new_precent}%", end="\r")
+ self.last_precent = new_precent
+
+def get_file_name(headers):
+ return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
+
+def get_sha256(filename):
+ sha256_hash = hashlib.sha256()
+ with open(filename,"rb") as f:
+ for byte_block in iter(lambda: f.read(4096),b""):
+ sha256_hash.update(byte_block)
+ file_checksum = sha256_hash.hexdigest()
+ return file_checksum
+ return
+
+def download_image_http(board: str, dest_folder: str, redownload: bool = False):
+ url = board["url"]
+ checksum = board["checksum"]
+
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ print('created temporary directory', tmpdirname)
+ temp_file_name = os.path.join(tmpdirname, "image.xz")
+ temp_file_checksum = os.path.join(tmpdirname, "checksum.sha256")
+
+ for r in range(RETRY):
+ try:
+ # Get sha and confirm its the right image
+ download_progress = DownloadProgress()
+ _, headers_checksum = urllib.request.urlretrieve(checksum, temp_file_checksum, download_progress.show_progress)
+ file_name_checksum = get_file_name(headers_checksum)
+
+ checksum_data = None
+ with open(temp_file_checksum, 'r') as f:
+ checksum_data = f.read()
+
+ checksum_data_parsed = [x.strip() for x in checksum_data.split()]
+ online_checksum = checksum_data_parsed[0]
+ file_name_from_checksum = checksum_data_parsed[1]
+ dest_file_name = os.path.join(dest_folder, file_name_from_checksum)
+ print(f"Downloading {dest_file_name}")
+
+ if os.path.isfile(dest_file_name):
+ file_checksum = get_sha256(dest_file_name)
+ if file_checksum == online_checksum:
+ # We got file and checksum is right
+ return
+ # Get the file
+ download_progress = DownloadProgress()
+ _, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress)
+
+ file_name = get_file_name(headers) 
+ file_checksum = get_sha256(temp_file_name)
+ if file_checksum != online_checksum:
+ print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}')
+ continue
+ ensure_dir(os.path.dirname(dest_file_name))
+ shutil.move(temp_file_name, dest_file_name)
+
+ except Exception as e:
+ if r < 2:
+ print(f'Failed. Attempt # {r + 1}, got: {e}')
+ else:
+ print('Error encoutered at {RETRY} attempt')
+ print(e)
+ else:
+ print(f"Success: {temp_file_name}")
+ break
+ return
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O')
+ parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+ parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+ args = parser.parse_args()
+
+ images = read_images()
+
+ base_board = os.environ.get("BASE_BOARD", None)
+ base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
+
+ if base_board is not None and base_board in images["images"]:
+ if images["images"][base_board]["type"] == "http":
+ download_image_http(images["images"][base_board], base_image_path)
+ elif images["images"][base_board]["type"] == "torrent":
+ print("Error: Torrent not implemented")
+ exit(1)
+ else:
+ print("Error: Unsupported image download type")
+ exit(1)
+
+ print("Done")

src/base_image_downloader_wrapper.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set +x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "$DIR/argparse.bash" || exit 1
+argparse "$@" <<EOF || exit 1
+
+parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+EOF
+
+if [ -z "${CUSTOM_PI_OS_PATH}" ];then
+ echo "Error: you must have \${CUSTOM_PI_OS_PATH} set"
+ exit 1
+fi
+
+# source "${DIST_PATH}/config"
+source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
+
+python3 ${CUSTOM_PI_OS_PATH}/base_image_downloader.py "${WORKSPACE_SUFFIX}"
+

src/build

@@ -18,7 +18,7 @@ CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 source ${CUSTOM_PI_OS_PATH}/config ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
-[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios
+[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}
 EOF
 
 if [ "$LOG" != "no" ]; then

src/common.sh

@@ -554,3 +554,36 @@ function set_config_var() {
  # See https://github.com/RPi-Distro/raspi-config/blob/master/raspi-config#L231
  raspi-config nonint set_config_var $1 $2 /boot/config.txt
 }
+
+
+function load_module_config() {
+ # Takes a comma seprated modules list, and exports the environment variables for it
+ MODULES_AFTER=$1
+ for module in $(echo "${MODULES_AFTER}" | tr "," "\n")
+ do
+ if [ -d "${DIST_PATH}/modules/${module}" ]; then
+ export MODULE_PATH="${DIST_PATH}/modules/${module}"
+ elif [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
+ export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
+ fi
+
+ echo "loading $module config at ${MODULE_PATH}/config"
+ if [ -f "${MODULE_PATH}/config" ]; then
+ source "${MODULE_PATH}/config"
+ else
+ echo "WARNING: module ${module} has no config file"
+ fi
+
+ ###############################################################################
+ # Print and export the final configuration.
+
+ echo "================================================================"
+ echo "Using the following config:"
+ module_up=${module^^} module_up=${module_up//-/_}_
+
+ # Export variables that satisfy the $module_up prefix
+ while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
+
+ echo "================================================================"
+ done
+}

src/config

@@ -1,4 +1,5 @@
 CONFIG_DIR=$(dirname $(realpath -s "${BASH_SOURCE}"))
+source ${CUSTOM_PI_OS_PATH}/common.sh
 
 WORKSPACE_POSTFIX=
 
@@ -79,31 +80,16 @@ TMP="${MODULES//(/,}"
 TMP="${TMP// /}"
 MODULES_LIST="${TMP//)/,}"
 
-for module in $(echo "${MODULES_LIST}" | tr "," "\n")
-do
- if [ -d "${DIST_PATH}/modules/${module}" ]; then
- export MODULE_PATH="${DIST_PATH}/modules/${module}"
- elif [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
- export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
- fi
-
- echo "loading $module config at ${MODULE_PATH}/config"
- if [ -f "${MODULE_PATH}/config" ]; then
- source "${MODULE_PATH}/config"
- else
- echo "WARNING: module ${module} has no config file"
- fi
-
- ###############################################################################
- # Print and export the final configuration.
-
- echo "================================================================"
- echo "Using the following config:"
- module_up=${module^^} module_up=${module_up//-/_}_
-
- # Export variables that satisfy the $module_up prefix
- while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
-
- echo "================================================================"
-done
 
+# Base workspace is special, it has to be sourced before the base module, so remote modules could be calcualted
+[ -n "$BASE_WORKSPACE" ] || BASE_WORKSPACE=${DIST_PATH}/workspace$WORKSPACE_POSTFIX
+# [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
+[ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
+
+export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
+# Remote modules and meta modulese go in first if they want to change standard behaviour
+if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+ source "${REMOTE_AND_META_CONFIG}"
+fi
+
+load_module_config "${MODULES_LIST}"

src/custompios

@@ -179,7 +179,20 @@ pushd $BASE_WORKSPACE
  # execute the base chroot script
  ### execute_chroot_script $BASE_SCRIPT_PATH $BASE_CHROOT_SCRIPT_PATH
  CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
- python3 ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" ${CHROOT_SCRIPT}
+ MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
+ MODULES_BEFORE="${MODULES}"
+ ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
+ if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+ echo "Sourcing remote and submodules config"
+ source "${REMOTE_AND_META_CONFIG}" ${@}
+
+ MODULES_AFTER=$(cat "${MODULES_AFTER_PATH}")
+ load_module_config "${MODULES_AFTER}"
+
+ else
+ echo "No remote and submodules config detected"
+ fi
+ echo $ARMBIAN_CONFIG_TXT_FILE
  export -f execute_chroot_script
  bash -x "${CHROOT_SCRIPT}"
 

src/execution_order.py

@@ -1,11 +1,11 @@
+#!/usr/bin/python3
 #a='base(octopi,a(b,c(a2)),mm)'
 import argparse
 import os
 import subprocess
 from get_remote_modules import get_remote_module
 from typing import TextIO, List, Tuple, Dict, Any
 
-
 def run_command(command: List[str], **kwargs: Dict[str, Any]):
  is_timeout = False
  p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
@@ -138,6 +138,8 @@ def handle_meta_modules(modules: List[Tuple[str,str]]) -> Tuple[List[Tuple[str,s
  parser = argparse.ArgumentParser(add_help=True, description='Parse and run CustomPiOS chroot modules')
  parser.add_argument('modules', type=str, help='A string showing how the modules should be called')
  parser.add_argument('output_script', type=str, help='path to output the chroot script master')
+ parser.add_argument('modules_after_path', nargs='?', default=None, type=str, help='path to output the chroot script master')
+ parser.add_argument('remote_and_meta_config_path', nargs='?', default=None, type=str, help='path to output the config script of remote modules and submodules')
  args = parser.parse_args()
 
  if os.path.isfile(args.output_script):
@@ -155,6 +157,27 @@ def handle_meta_modules(modules: List[Tuple[str,str]]) -> Tuple[List[Tuple[str,s
  for module, state in modules_execution_order:
  module_folder = modules_to_modules_folder[module]
  write_modules_scripts(module, state, module_folder, f)
+
+ # List all new modules add them in, then remove existing ones
+ list_new_modules = []
+ for module, state in modules_execution_order:
+ if module not in list_new_modules:
+ list_new_modules.append(module)
+ for module, state in initial_execution_order:
+ if module in list_new_modules:
+ list_new_modules.remove(module)
 
+ # TODO2: load configs from yaml
+ if args.modules_after_path is not None:
+ with open(args.modules_after_path, "w") as w:
+ w.write(",".join(list_new_modules))
+
+ with open(args.remote_and_meta_config_path, "w") as f:
+ for module in list_new_modules:
+ module_folder = modules_to_modules_folder[module]
+ module_config_path = os.path.join(module_folder, "config")
+ if os.path.isfile(module_config_path):
+ f.write(f"source {module_config_path}\n")
+
  os.chmod(args.output_script, 0o755)
 

src/modules/base/config

@@ -8,18 +8,24 @@ BASE_VERSION=1.5.0
 [ -n "$BASE_PRESCRIPT" ] || BASE_PRESCRIPT=
 [ -n "$BASE_POSTSCRIPT" ] || BASE_POSTSCRIPT=
 
+# Board and OS
+[ -n "$BASE_BOARD" ] || BASE_BOARD="raspberrypi"
+[ -n "$BASE_OS" ] || BASE_OS="debian_bookworm"
+
+# TODO: UNIFY AND MAKE A SINGLE IMAGE FLOW, ATM THERE IS THE OLD AND NEW FLOW TOGETHER, UBUTU CAN GO IN ITS OWN
+
 #[ -n "$BASE_SCRIPT_PATH" ] || BASE_SCRIPT_PATH=$CONFIG_DIR
-[ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image
+if [ "${BASE_BOARD}" == "raspberrypiarmhf" ] && [ "${BASE_BOARD}" == "raspberrypiarm64" ]; then
+ [ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image
+else
+ [ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image-"${BASE_BOARD}"
+ BASE_ZIP_IMG=`ls -t $BASE_IMAGE_PATH/*.{zip,7z,xz} | head -n 1`
+fi
 [ -n "$BASE_IMAGE_RASPBIAN" ] || BASE_IMAGE_RASPBIAN=yes
 
 # Distro
 [ -n "$BASE_DISTRO" ] || BASE_DISTRO=raspbian
 
-# Board and OS
-[ -n "$BASE_BOARD" ] || BASE_BOARD="raspberrypi"
-[ -n "$BASE_OS" ] || BASE_OS="debian_bookworm"
-
-
 # Note: Set BASE_ZIP_IMG relative to the distro/src/workspace directory to pass a custom named file or an already extracted '.img'-file.
 if [ "${BASE_DISTRO}" == "ubuntu" ]; then
  # Default image ubuntu
@@ -33,7 +39,7 @@ if [ "${BASE_DISTRO}" == "ubuntu" ]; then
  [ -n "$BASE_USER_PASSWORD" ] || BASE_USER_PASSWORD=ubuntu
 else
  # Default image raspbian
- if [ "${BASE_DISTRO}" == "raspios64" ]; then
+ if [ "${BASE_DISTRO}" == "raspios64" ] && [ "${BASE_BOARD}" == "raspberrypiarmhf" ] && [ "${BASE_BOARD}" == "raspberrypiarm64" ]; then
  [ -n "$BASE_ZIP_IMG" ] || BASE_ZIP_IMG=`ls -t $BASE_IMAGE_PATH/*-{raspbian,raspios}-*-arm64-*.{zip,7z,xz} | head -n 1`
  else
  [ -n "$BASE_ZIP_IMG" ] || BASE_ZIP_IMG=`ls -t $BASE_IMAGE_PATH/*-{raspbian,raspios}*.{zip,7z,xz} | head -n 1`
@@ -50,10 +56,6 @@ fi
 [ -n "$BASE_RELEASE_IMG_NAME" ] || BASE_RELEASE_IMG_NAME=default
 [ -n "$BASE_RELEASE_ZIP_NAME" ] || BASE_RELEASE_ZIP_NAME=default
 
-[ -n "$BASE_WORKSPACE" ] || BASE_WORKSPACE=${DIST_PATH}/workspace$WORKSPACE_POSTFIX
-# [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
-[ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
-
 if [ "${BASE_DISTRO}" == "ubuntu" ]; then
  [ -n "${BASE_BOOT_MOUNT_PATH}" ] || BASE_BOOT_MOUNT_PATH=boot/firmware
 else

src/modules/base/meta

@@ -10,9 +10,9 @@ export LC_ALL=C
 
 FINAL_MODULES=()
 
-if [ "${BASE_BOARD}" == "armbian" ]; then
+if [[ "${BASE_BOARD}" = armbian* ]]; then
  FINAL_MODULES+=("armbian")
-elif [ "${BASE_BOARD}" == "orange" ]; then
+elif [[ "${BASE_BOARD}" = orange* ]]; then
  FINAL_MODULES+=("orange")
 fi
 

src/modules/network/meta

@@ -10,9 +10,9 @@ export LC_ALL=C
 
 FINAL_MODULES=()
 
-if [ "${BASE_BOARD}" == "armbian" ]; then
+if [[ "${BASE_BOARD}" = armbian* ]]; then
  FINAL_MODULES+=("armbian_net")
-elif [ "${BASE_BOARD}" == "orange" ]; then
+elif [[ "${BASE_BOARD}" = orange* ]]; then
  FINAL_MODULES+=("orange_net")
 fi