Major rework for CustomPiOs v2 - add config loading of meta and remot…

…e modules, get meta modules and images name to match #214

src/base_image_downloader.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+import argparse
+import yaml
+import os
+import urllib.request
+import tempfile
+import hashlib
+import shutil
+import re
+PRECENT_PROGRESS_SIZE = 5
+
+class ChecksumFailException(Exception):
+    pass
+
+IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
+RETRY = 3
+
+def ensure_dir(d, chmod=0o777):
+    """
+    Ensures a folder exists.
+    Returns True if the folder already exists
+    """
+    if not os.path.exists(d):
+        os.makedirs(d, chmod)
+        os.chmod(d, chmod)
+        return False
+    return True
+
+def read_images():
+    if not os.path.isfile(IMAGES_CONFIG):
+        raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
+    with open(IMAGES_CONFIG,'r') as f:
+        output = yaml.safe_load(f)
+        return output
+
+class DownloadProgress:
+    last_precent: float = 0
+    def show_progress(self, block_num, block_size, total_size):
+        new_precent = round(block_num * block_size / total_size * 100, 1)
+        if new_precent > self.last_precent + PRECENT_PROGRESS_SIZE:
+            print(f"{new_precent}%", end="\r")
+            self.last_precent = new_precent
+
+def get_file_name(headers):
+    return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
+
+def get_sha256(filename):
+    sha256_hash = hashlib.sha256()
+    with open(filename,"rb") as f:
+        for byte_block in iter(lambda: f.read(4096),b""):
+            sha256_hash.update(byte_block)
+        file_checksum = sha256_hash.hexdigest()
+        return file_checksum
+    return
+
+def download_image_http(board: str, dest_folder: str, redownload: bool = False):
+    url = board["url"]
+    checksum = board["checksum"]
+
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        print('created temporary directory', tmpdirname)
+        temp_file_name = os.path.join(tmpdirname, "image.xz")
+        temp_file_checksum = os.path.join(tmpdirname, "checksum.sha256")
+
+        for r in range(RETRY):
+            try:
+                # Get sha and confirm its the right image
+                download_progress = DownloadProgress()
+                _, headers_checksum = urllib.request.urlretrieve(checksum, temp_file_checksum, download_progress.show_progress)
+                file_name_checksum = get_file_name(headers_checksum)
+
+                checksum_data = None
+                with open(temp_file_checksum, 'r') as f:
+                    checksum_data = f.read()
+
+                checksum_data_parsed = [x.strip() for x in checksum_data.split()]
+                online_checksum = checksum_data_parsed[0]
+                file_name_from_checksum = checksum_data_parsed[1]
+                dest_file_name = os.path.join(dest_folder, file_name_from_checksum)
+                print(f"Downloading {dest_file_name}")
+
+                if os.path.isfile(dest_file_name):
+                    file_checksum = get_sha256(dest_file_name)
+                    if file_checksum == online_checksum:
+                        # We got file and checksum is right
+                        return
+                # Get the file
+                download_progress = DownloadProgress()
+                _, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress)
+
+                file_name = get_file_name(headers)                        
+                file_checksum = get_sha256(temp_file_name)
+                if file_checksum != online_checksum:
+                    print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}')
+                    continue
+                ensure_dir(os.path.dirname(dest_file_name))
+                shutil.move(temp_file_name, dest_file_name)
+
+            except Exception as e:
+                if r < 2:
+                    print(f'Failed. Attempt # {r + 1}, got: {e}')
+                else:
+                    print('Error encoutered at {RETRY} attempt')
+                    print(e)
+            else:
+                print(f"Success: {temp_file_name}")
+                break
+    return
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O')
+    parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+    parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+    args = parser.parse_args()
+
+    images = read_images()
+
+    base_board = os.environ.get("BASE_BOARD", None)
+    base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
+
+    if base_board is not None and base_board in images["images"]:
+        if images["images"][base_board]["type"] == "http":
+            download_image_http(images["images"][base_board], base_image_path)
+        elif images["images"][base_board]["type"] == "torrent":
+            print("Error: Torrent not implemented")
+            exit(1)
+        else:
+            print("Error: Unsupported image download type")
+            exit(1)
+
+    print("Done")

src/base_image_downloader_wrapper.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set +x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "$DIR/argparse.bash" || exit 1
+argparse "$@" <<EOF || exit 1
+
+parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+EOF
+
+if [ -z "${CUSTOM_PI_OS_PATH}" ];then
+    echo "Error: you must have \${CUSTOM_PI_OS_PATH} set"
+    exit 1
+fi
+
+# source "${DIST_PATH}/config"
+source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
+
+python3 ${CUSTOM_PI_OS_PATH}/base_image_downloader.py "${WORKSPACE_SUFFIX}"
+

src/build

@@ -18,7 +18,7 @@ CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 source ${CUSTOM_PI_OS_PATH}/config ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
-[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios
+[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}
 EOF
 
 if [ "$LOG" != "no" ]; then

src/common.sh

@@ -554,3 +554,36 @@ function set_config_var() {
   # See https://github.com/RPi-Distro/raspi-config/blob/master/raspi-config#L231
   raspi-config nonint set_config_var $1 $2 /boot/config.txt
 }
+
+
+function load_module_config() {
+  # Takes a comma seprated modules list, and exports the environment variables for it
+  MODULES_AFTER=$1
+  for module in $(echo "${MODULES_AFTER}" | tr "," "\n")
+  do
+      if [ -d "${DIST_PATH}/modules/${module}" ]; then
+          export MODULE_PATH="${DIST_PATH}/modules/${module}"
+      elif   [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
+          export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
+      fi
+
+      echo "loading $module config at ${MODULE_PATH}/config"
+      if [ -f "${MODULE_PATH}/config" ]; then
+          source "${MODULE_PATH}/config"
+      else
+          echo "WARNING: module ${module} has no config file"
+      fi
+
+      ###############################################################################
+      # Print and export the final configuration.
+
+      echo "================================================================"
+      echo "Using the following config:"
+      module_up=${module^^} module_up=${module_up//-/_}_
+
+      # Export variables that satisfy the $module_up prefix
+      while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
+
+      echo "================================================================"
+  done
+}

src/config

@@ -1,4 +1,5 @@
 CONFIG_DIR=$(dirname $(realpath -s "${BASH_SOURCE}"))
+source ${CUSTOM_PI_OS_PATH}/common.sh
 
 WORKSPACE_POSTFIX=
 
@@ -79,31 +80,16 @@ TMP="${MODULES//(/,}"
 TMP="${TMP// /}"
 MODULES_LIST="${TMP//)/,}"
 
-for module in $(echo "${MODULES_LIST}" | tr "," "\n")
-do
-    if [ -d "${DIST_PATH}/modules/${module}" ]; then
-        export MODULE_PATH="${DIST_PATH}/modules/${module}"
-    elif   [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
-        export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
-    fi
-
-    echo "loading $module config at ${MODULE_PATH}/config"
-    if [ -f "${MODULE_PATH}/config" ]; then
-        source "${MODULE_PATH}/config"
-    else
-        echo "WARNING: module ${module} has no config file"
-    fi
-
-    ###############################################################################
-    # Print and export the final configuration.
-
-    echo "================================================================"
-    echo "Using the following config:"
-    module_up=${module^^} module_up=${module_up//-/_}_
-
-    # Export variables that satisfy the $module_up prefix
-    while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
-
-    echo "================================================================"
-done
 
+# Base workspace is special, it has to be sourced before the base module, so remote modules could be calcualted
+[ -n "$BASE_WORKSPACE" ] || BASE_WORKSPACE=${DIST_PATH}/workspace$WORKSPACE_POSTFIX
+# [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
+[ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
+
+export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
+# Remote modules and meta modulese go in first if they want to change standard behaviour
+if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+  source "${REMOTE_AND_META_CONFIG}"
+fi
+
+load_module_config "${MODULES_LIST}"

src/custompios

@@ -179,7 +179,20 @@ pushd $BASE_WORKSPACE
     # execute the base chroot script
     ### execute_chroot_script $BASE_SCRIPT_PATH $BASE_CHROOT_SCRIPT_PATH
     CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
-    python3 ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" ${CHROOT_SCRIPT}
+    MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
+    MODULES_BEFORE="${MODULES}"
+    ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
+    if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+      echo "Sourcing remote and submodules config"
+      source "${REMOTE_AND_META_CONFIG}" ${@}
+
+      MODULES_AFTER=$(cat "${MODULES_AFTER_PATH}")
+      load_module_config "${MODULES_AFTER}"
+
+    else
+      echo "No remote and submodules config detected"
+    fi
+    echo $ARMBIAN_CONFIG_TXT_FILE
     export -f execute_chroot_script
     bash -x "${CHROOT_SCRIPT}"
 

src/execution_order.py

@@ -1,11 +1,11 @@
+#!/usr/bin/python3
 #a='base(octopi,a(b,c(a2)),mm)'
 import argparse
 import os
 import subprocess
 from get_remote_modules import get_remote_module
 from typing import TextIO, List, Tuple, Dict, Any
 
-
 def run_command(command: List[str], **kwargs: Dict[str, Any]):
     is_timeout = False
     p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
@@ -138,6 +138,8 @@ def handle_meta_modules(modules: List[Tuple[str,str]]) -> Tuple[List[Tuple[str,s
     parser = argparse.ArgumentParser(add_help=True, description='Parse and run CustomPiOS chroot modules')
     parser.add_argument('modules', type=str, help='A string showing how the modules should be called')
     parser.add_argument('output_script', type=str, help='path to output the chroot script master')
+    parser.add_argument('modules_after_path', nargs='?', default=None, type=str, help='path to output the chroot script master')
+    parser.add_argument('remote_and_meta_config_path', nargs='?', default=None, type=str, help='path to output the config script of remote modules and submodules')
     args = parser.parse_args()
 
     if os.path.isfile(args.output_script):
@@ -155,6 +157,27 @@ def handle_meta_modules(modules: List[Tuple[str,str]]) -> Tuple[List[Tuple[str,s
         for module, state in modules_execution_order:
             module_folder = modules_to_modules_folder[module]
             write_modules_scripts(module, state, module_folder, f)
+
+        # List all new modules add them in, then remove existing ones
+        list_new_modules = []
+        for module, state in modules_execution_order:
+            if module not in list_new_modules:
+                list_new_modules.append(module)
+        for module, state in initial_execution_order:
+            if module in list_new_modules:
+                list_new_modules.remove(module)
 
+    # TODO2: load configs from yaml
+    if args.modules_after_path is not None:
+        with open(args.modules_after_path, "w") as w:
+            w.write(",".join(list_new_modules))
+
+    with open(args.remote_and_meta_config_path, "w") as f:
+        for module in list_new_modules:
+            module_folder = modules_to_modules_folder[module]
+            module_config_path = os.path.join(module_folder, "config")
+            if os.path.isfile(module_config_path):
+                f.write(f"source {module_config_path}\n")
+
     os.chmod(args.output_script, 0o755)
 

src/modules/base/config

@@ -8,18 +8,24 @@ BASE_VERSION=1.5.0
 [ -n "$BASE_PRESCRIPT" ] || BASE_PRESCRIPT=
 [ -n "$BASE_POSTSCRIPT" ] || BASE_POSTSCRIPT=
 
+# Board and OS
+[ -n "$BASE_BOARD" ] || BASE_BOARD="raspberrypi"
+[ -n "$BASE_OS" ] || BASE_OS="debian_bookworm"
+
+# TODO: UNIFY AND MAKE A SINGLE IMAGE FLOW, ATM THERE IS THE OLD AND NEW FLOW TOGETHER, UBUTU CAN GO IN ITS OWN
+
 #[ -n "$BASE_SCRIPT_PATH" ] || BASE_SCRIPT_PATH=$CONFIG_DIR
-[ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image
+if [ "${BASE_BOARD}" == "raspberrypiarmhf" ] && [ "${BASE_BOARD}" == "raspberrypiarm64" ]; then
+    [ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image
+else
+    [ -n "$BASE_IMAGE_PATH" ] || BASE_IMAGE_PATH=${DIST_PATH}/image-"${BASE_BOARD}"
+    BASE_ZIP_IMG=`ls -t $BASE_IMAGE_PATH/*.{zip,7z,xz} | head -n 1`
+fi
 [ -n "$BASE_IMAGE_RASPBIAN" ] || BASE_IMAGE_RASPBIAN=yes
 
 # Distro
 [ -n "$BASE_DISTRO" ] || BASE_DISTRO=raspbian
 
-# Board and OS
-[ -n "$BASE_BOARD" ] || BASE_BOARD="raspberrypi"
-[ -n "$BASE_OS" ] || BASE_OS="debian_bookworm"
-
-
 # Note: Set BASE_ZIP_IMG relative to the distro/src/workspace directory to pass a custom named file or an already extracted '.img'-file.
 if [ "${BASE_DISTRO}" = "ubuntu" ]; then
     # Default image ubuntu
@@ -33,7 +39,7 @@ if [ "${BASE_DISTRO}" = "ubuntu" ]; then
     [ -n "$BASE_USER_PASSWORD" ] || BASE_USER_PASSWORD=ubuntu
 else
     # Default image raspbian
-    if [ "${BASE_DISTRO}" = "raspios64" ]; then
+    if [ "${BASE_DISTRO}" == "raspios64" ] && [ "${BASE_BOARD}" == "raspberrypiarmhf" ] && [ "${BASE_BOARD}" == "raspberrypiarm64" ]; then
         [ -n "$BASE_ZIP_IMG" ] || BASE_ZIP_IMG=$(ls -t $BASE_IMAGE_PATH/*-{raspbian,raspios}-*-arm64-*.{zip,7z,xz} | head -n 1)
     else
         [ -n "$BASE_ZIP_IMG" ] || BASE_ZIP_IMG=$(ls -t $BASE_IMAGE_PATH/*-{raspbian,raspios}*.{zip,7z,xz} | head -n 1)

src/modules/base/meta

@@ -10,9 +10,9 @@ export LC_ALL=C
 
 FINAL_MODULES=()
 
-if [ "${BASE_BOARD}" == "armbian" ]; then
+if [[ "${BASE_BOARD}" = armbian* ]]; then
     FINAL_MODULES+=("armbian")
-elif [ "${BASE_BOARD}" == "orange" ]; then
+elif [[ "${BASE_BOARD}" = orange* ]]; then
     FINAL_MODULES+=("orange")
 fi
 

src/modules/network/meta

@@ -10,9 +10,9 @@ export LC_ALL=C
 
 FINAL_MODULES=()
 
-if [ "${BASE_BOARD}" == "armbian" ]; then
+if [[ "${BASE_BOARD}" = armbian* ]]; then
     FINAL_MODULES+=("armbian_net")
-elif [ "${BASE_BOARD}" == "orange" ]; then
+elif [[ "${BASE_BOARD}" = orange* ]]; then
     FINAL_MODULES+=("orange_net")
 fi
 

src/modules_remote.yml

@@ -64,3 +64,4 @@ remotes:
     repo: https://github.com/mainsail-crew/MainsailOS.git
     type: git
     tag: develop
+    auto_update: true