Add base board config generation from yaml, refactor and add a common…

….py and typing #214

src/base_image_downloader.py

@@ -7,12 +7,13 @@
 import hashlib
 import shutil
 import re
+from typing import Dict, Any, Optional, cast
+from common import get_image_config, read_images
 PRECENT_PROGRESS_SIZE = 5
 
 class ChecksumFailException(Exception):
  pass
 
-IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
 RETRY = 3
 
 def ensure_dir(d, chmod=0o777):
@@ -26,13 +27,6 @@ def ensure_dir(d, chmod=0o777):
  return False
  return True
 
-def read_images():
- if not os.path.isfile(IMAGES_CONFIG):
- raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
- with open(IMAGES_CONFIG,'r') as f:
- output = yaml.safe_load(f)
- return output
-
 class DownloadProgress:
  last_precent: float = 0
  def show_progress(self, block_num, block_size, total_size):
@@ -53,7 +47,7 @@ def get_sha256(filename):
  return file_checksum
  return
 
-def download_image_http(board: str, dest_folder: str, redownload: bool = False):
+def download_image_http(board: Dict[str, Any], dest_folder: str, redownload: bool = False):
  url = board["url"]
  checksum = board["checksum"]
 
@@ -118,14 +112,21 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
  base_board = os.environ.get("BASE_BOARD", None)
  base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
 
- if base_board is not None and base_board in images["images"]:
- if images["images"][base_board]["type"] == "http":
- download_image_http(images["images"][base_board], base_image_path)
- elif images["images"][base_board]["type"] == "torrent":
+ if base_image_path is None:
+ print(f'Error: did not find image config file')
+ exit(1)
+ cast(str, base_image_path)
+
+ image_config = get_image_config()
+ if image_config is not None:
+ if image_config["type"] == "http":
+ download_image_http(image_config, base_image_path)
+ elif image_config["type"] == "torrent":
  print("Error: Torrent not implemented")
  exit(1)
  else:
- print("Error: Unsupported image download type")
+ print(f'Error: Unsupported image download type: {image_config["type"]}')
  exit(1)
+
 
  print("Done")

src/build

@@ -10,12 +10,16 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
 
 define SCRIPT <<'EOF'
 BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
+export EXTRA_BAORD_CONFIG=$(mktemp)
+${BUILD_SCRIPT_PATH}/generate_board_config.py "${EXTRA_BAORD_CONFIG}"
+echo "Temp source file: ${EXTRA_BAORD_CONFIG}"
+
 source ${BUILD_SCRIPT_PATH}/common.sh
 install_cleanup_trap
 
 CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 
-source ${CUSTOM_PI_OS_PATH}/config ${@}
+source ${CUSTOM_PI_OS_PATH}/config ${1} "${EXTRA_BAORD_CONFIG}" ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
 [ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}

src/common.py

@@ -0,0 +1,23 @@
+""" Common functions between CustomPiOS python scripts"""
+from typing import Dict, Any, Optional, cast
+import yaml
+import os
+
+IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
+
+def read_images():
+ if not os.path.isfile(IMAGES_CONFIG):
+ raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
+ with open(IMAGES_CONFIG,'r') as f:
+ output = yaml.safe_load(f)
+ return output
+
+def get_image_config() -> Optional[Dict["str", Any]]:
+ images = read_images()
+
+ base_board = os.environ.get("BASE_BOARD", None)
+ base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
+
+ if base_board is not None and base_board in images["images"]:
+ return images["images"][base_board]
+ return None

src/config

@@ -7,6 +7,8 @@ export BUILD_VARIANT=""
 BUILD_VARIANT="$1"
 : ${BUILD_VARIANT:=default}
 
+EXTRA_BAORD_CONFIG=$2
+
 export BUILD_FLAVOR=""
 # Disable flavor system
 #BUILD_FLAVOR="$1"
@@ -86,6 +88,13 @@ MODULES_LIST="${TMP//)/,}"
 # [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
 [ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
 
+# Import remote and submodules config
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+ source "${EXTRA_BAORD_CONFIG}"
+else
+ echo "Note: Not sourceing board config"
+fi
+
 export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
 # Remote modules and meta modulese go in first if they want to change standard behaviour
 if [ -f "${REMOTE_AND_META_CONFIG}" ]; then

src/custompios

@@ -108,6 +108,11 @@ fi
 mkdir -p $BASE_WORKSPACE
 mkdir -p $BASE_MOUNT_PATH
 
+# This is already genrated at "build" sourced in "config", but copying here mostly for debug
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+ mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config
+fi
+
 # Clean exported artifacts from other builds
 rm -rf "${BASE_WORKSPACE}"/*.tar.gz
 

src/requirements-devel.txt

@@ -0,0 +1 @@
+types-PyYAML