Meta modules support #214

src/Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  python3-distutils \
  python3-dev \
  python3-git \
+ python3-yaml \
  binfmt-support \
  qemu-system \
  qemu-user \

src/base_image_downloader.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+import argparse
+import yaml
+import os
+import urllib.request
+import tempfile
+import hashlib
+import shutil
+import re
+PRECENT_PROGRESS_SIZE = 5
+
+class ChecksumFailException(Exception):
+ pass
+
+IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
+RETRY = 3
+
+def ensure_dir(d, chmod=0o777):
+ """
+ Ensures a folder exists.
+ Returns True if the folder already exists
+ """
+ if not os.path.exists(d):
+ os.makedirs(d, chmod)
+ os.chmod(d, chmod)
+ return False
+ return True
+
+def read_images():
+ if not os.path.isfile(IMAGES_CONFIG):
+ raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
+ with open(IMAGES_CONFIG,'r') as f:
+ output = yaml.safe_load(f)
+ return output
+
+class DownloadProgress:
+ last_precent: float = 0
+ def show_progress(self, block_num, block_size, total_size):
+ new_precent = round(block_num * block_size / total_size * 100, 1)
+ if new_precent > self.last_precent + PRECENT_PROGRESS_SIZE:
+ print(f"{new_precent}%", end="\r")
+ self.last_precent = new_precent
+
+def get_file_name(headers):
+ return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
+
+def get_sha256(filename):
+ sha256_hash = hashlib.sha256()
+ with open(filename,"rb") as f:
+ for byte_block in iter(lambda: f.read(4096),b""):
+ sha256_hash.update(byte_block)
+ file_checksum = sha256_hash.hexdigest()
+ return file_checksum
+ return
+
+def download_image_http(board: str, dest_folder: str, redownload: bool = False):
+ url = board["url"]
+ checksum = board["checksum"]
+
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ print('created temporary directory', tmpdirname)
+ temp_file_name = os.path.join(tmpdirname, "image.xz")
+ temp_file_checksum = os.path.join(tmpdirname, "checksum.sha256")
+
+ for r in range(RETRY):
+ try:
+ # Get sha and confirm its the right image
+ download_progress = DownloadProgress()
+ _, headers_checksum = urllib.request.urlretrieve(checksum, temp_file_checksum, download_progress.show_progress)
+ file_name_checksum = get_file_name(headers_checksum)
+
+ checksum_data = None
+ with open(temp_file_checksum, 'r') as f:
+ checksum_data = f.read()
+
+ checksum_data_parsed = [x.strip() for x in checksum_data.split()]
+ online_checksum = checksum_data_parsed[0]
+ file_name_from_checksum = checksum_data_parsed[1]
+ dest_file_name = os.path.join(dest_folder, file_name_from_checksum)
+ print(f"Downloading {dest_file_name}")
+
+ if os.path.isfile(dest_file_name):
+ file_checksum = get_sha256(dest_file_name)
+ if file_checksum == online_checksum:
+ # We got file and checksum is right
+ return
+ # Get the file
+ download_progress = DownloadProgress()
+ _, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress)
+
+ file_name = get_file_name(headers) 
+ file_checksum = get_sha256(temp_file_name)
+ if file_checksum != online_checksum:
+ print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}')
+ continue
+ ensure_dir(os.path.dirname(dest_file_name))
+ shutil.move(temp_file_name, dest_file_name)
+
+ except Exception as e:
+ if r < 2:
+ print(f'Failed. Attempt # {r + 1}, got: {e}')
+ else:
+ print('Error encoutered at {RETRY} attempt')
+ print(e)
+ else:
+ print(f"Success: {temp_file_name}")
+ break
+ return
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O')
+ parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+ parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+ args = parser.parse_args()
+
+ images = read_images()
+
+ base_board = os.environ.get("BASE_BOARD", None)
+ base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
+
+ if base_board is not None and base_board in images["images"]:
+ if images["images"][base_board]["type"] == "http":
+ download_image_http(images["images"][base_board], base_image_path)
+ elif images["images"][base_board]["type"] == "torrent":
+ print("Error: Torrent not implemented")
+ exit(1)
+ else:
+ print("Error: Unsupported image download type")
+ exit(1)
+
+ print("Done")

src/base_image_downloader_wrapper.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set +x
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "$DIR/argparse.bash" || exit 1
+argparse "$@" <<EOF || exit 1
+
+parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
+parser.add_argument('-s', '--sha256', action='store_true', help='Create a sha256 hash for the .img file in .sha256')
+EOF
+
+if [ -z "${CUSTOM_PI_OS_PATH}" ];then
+ echo "Error: you must have \${CUSTOM_PI_OS_PATH} set"
+ exit 1
+fi
+
+# source "${DIST_PATH}/config"
+source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
+
+python3 ${CUSTOM_PI_OS_PATH}/base_image_downloader.py "${WORKSPACE_SUFFIX}"
+

src/build

@@ -18,7 +18,7 @@ CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 source ${CUSTOM_PI_OS_PATH}/config ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
-[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios
+[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}
 EOF
 
 if [ "$LOG" != "no" ]; then

src/common.sh

@@ -554,3 +554,36 @@ function set_config_var() {
  # See https://github.com/RPi-Distro/raspi-config/blob/master/raspi-config#L231
  raspi-config nonint set_config_var $1 $2 /boot/config.txt
 }
+
+
+function load_module_config() {
+ # Takes a comma seprated modules list, and exports the environment variables for it
+ MODULES_AFTER=$1
+ for module in $(echo "${MODULES_AFTER}" | tr "," "\n")
+ do
+ if [ -d "${DIST_PATH}/modules/${module}" ]; then
+ export MODULE_PATH="${DIST_PATH}/modules/${module}"
+ elif [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
+ export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
+ fi
+
+ echo "loading $module config at ${MODULE_PATH}/config"
+ if [ -f "${MODULE_PATH}/config" ]; then
+ source "${MODULE_PATH}/config"
+ else
+ echo "WARNING: module ${module} has no config file"
+ fi
+
+ ###############################################################################
+ # Print and export the final configuration.
+
+ echo "================================================================"
+ echo "Using the following config:"
+ module_up=${module^^} module_up=${module_up//-/_}_
+
+ # Export variables that satisfy the $module_up prefix
+ while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
+
+ echo "================================================================"
+ done
+}

src/config

@@ -1,4 +1,5 @@
 CONFIG_DIR=$(dirname $(realpath -s "${BASH_SOURCE}"))
+source ${CUSTOM_PI_OS_PATH}/common.sh
 
 WORKSPACE_POSTFIX=
 
@@ -79,31 +80,16 @@ TMP="${MODULES//(/,}"
 TMP="${TMP// /}"
 MODULES_LIST="${TMP//)/,}"
 
-for module in $(echo "${MODULES_LIST}" | tr "," "\n")
-do
- if [ -d "${DIST_PATH}/modules/${module}" ]; then
- export MODULE_PATH="${DIST_PATH}/modules/${module}"
- elif [ -d "${CUSTOM_PI_OS_PATH}/modules/${module}" ]; then
- export MODULE_PATH="${CUSTOM_PI_OS_PATH}/modules/${module}"
- fi
-
- echo "loading $module config at ${MODULE_PATH}/config"
- if [ -f "${MODULE_PATH}/config" ]; then
- source "${MODULE_PATH}/config"
- else
- echo "WARNING: module ${module} has no config file"
- fi
-
- ###############################################################################
- # Print and export the final configuration.
-
- echo "================================================================"
- echo "Using the following config:"
- module_up=${module^^} module_up=${module_up//-/_}_
-
- # Export variables that satisfy the $module_up prefix
- while IFS= read -r var; do export "$var"; echo "$var"; done < <(compgen -A variable "$module_up")
-
- echo "================================================================"
-done
 
+# Base workspace is special, it has to be sourced before the base module, so remote modules could be calcualted
+[ -n "$BASE_WORKSPACE" ] || BASE_WORKSPACE=${DIST_PATH}/workspace$WORKSPACE_POSTFIX
+# [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
+[ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
+
+export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
+# Remote modules and meta modulese go in first if they want to change standard behaviour
+if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+ source "${REMOTE_AND_META_CONFIG}"
+fi
+
+load_module_config "${MODULES_LIST}"

src/custompios

@@ -179,7 +179,20 @@ pushd $BASE_WORKSPACE
  # execute the base chroot script
  ### execute_chroot_script $BASE_SCRIPT_PATH $BASE_CHROOT_SCRIPT_PATH
  CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
- python3 ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" ${CHROOT_SCRIPT}
+ MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
+ MODULES_BEFORE="${MODULES}"
+ ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
+ if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
+ echo "Sourcing remote and submodules config"
+ source "${REMOTE_AND_META_CONFIG}" ${@}
+
+ MODULES_AFTER=$(cat "${MODULES_AFTER_PATH}")
+ load_module_config "${MODULES_AFTER}"
+
+ else
+ echo "No remote and submodules config detected"
+ fi
+ echo $ARMBIAN_CONFIG_TXT_FILE
  export -f execute_chroot_script
  bash -x "${CHROOT_SCRIPT}"