Auto download images by base board, make python code a python package #…

…214

pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "custompios"
+version = "2.0.0"
+description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship."
+authors = ["Guy Sheffer <guysoft@gmail.com>"]
+license = "GPLv3"
+readme = "README.rst"
+packages = [
+# { include = "src/*" },
+ { include = "custompios_core", from = "src" }
+ ]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+GitPython = "^3.1.41"
+
+[tool.poetry.group.dev.dependencies]
+types-PyYAML = "^6.0.12.12"
+
+[tool.poetry.scripts]
+custompios_build = 'custompios_core.multi_build:main'
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

src/base_image_downloader_wrapper.sh

@@ -17,5 +17,5 @@ fi
 # source "${DIST_PATH}/config"
 source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
 
-python3 ${CUSTOM_PI_OS_PATH}/base_image_downloader.py "${WORKSPACE_SUFFIX}"
+python3 ${CUSTOM_PI_OS_PATH}/custompios_core/base_image_downloader.py "${WORKSPACE_SUFFIX}"
 

src/build

@@ -10,12 +10,16 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
 
 define SCRIPT <<'EOF'
 BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
+export EXTRA_BOARD_CONFIG=$(mktemp)
+${BUILD_SCRIPT_PATH}/custompios_core/generate_board_config.py "${EXTRA_BOARD_CONFIG}"
+echo "Temp source file: ${EXTRA_BOARD_CONFIG}"
+
 source ${BUILD_SCRIPT_PATH}/common.sh
 install_cleanup_trap
 
 CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
 
-source ${CUSTOM_PI_OS_PATH}/config ${@}
+source ${CUSTOM_PI_OS_PATH}/config "${1}" "${EXTRA_BOARD_CONFIG}" ${@}
 ${CUSTOM_PI_OS_PATH}/config_sanity
 
 [ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}

src/common.sh

@@ -150,10 +150,13 @@ function unpack() {
 }
 
 function detach_all_loopback(){
+ image_name=$1
  # Cleans up mounted loopback devices from the image name
  # NOTE: it might need a better way to grep for the image name, its might clash with other builds
  for img in $(losetup | grep $1 | awk '{ print $1 }' ); do
- if [ -f "${img}" ] || [ -b "${img}" ]; then
+ # test if the image name is a substring
+ if [ "${img}" != "$(printf '%s' "${img}" | sed 's/'"${image_name}"'//g')" ] && ([ -f "${img}" ] || [ -b "${img}" ]); then
+ echo "freeing up $img"
  losetup -d $img
  fi
  done

src/config

@@ -7,6 +7,8 @@ export BUILD_VARIANT=""
 BUILD_VARIANT="$1"
 : ${BUILD_VARIANT:=default}
 
+EXTRA_BAORD_CONFIG=$2
+
 export BUILD_FLAVOR=""
 # Disable flavor system
 #BUILD_FLAVOR="$1"
@@ -86,6 +88,13 @@ MODULES_LIST="${TMP//)/,}"
 # [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
 [ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
 
+# Import remote and submodules config
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+ source "${EXTRA_BAORD_CONFIG}"
+else
+ echo "Note: Not sourceing board config"
+fi
+
 export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
 # Remote modules and meta modulese go in first if they want to change standard behaviour
 if [ -f "${REMOTE_AND_META_CONFIG}" ]; then

src/custompios

@@ -108,6 +108,11 @@ fi
 mkdir -p $BASE_WORKSPACE
 mkdir -p $BASE_MOUNT_PATH
 
+# This is already genrated at "build" sourced in "config", but copying here mostly for debug
+if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
+ mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config
+fi
+
 # Clean exported artifacts from other builds
 rm -rf "${BASE_WORKSPACE}"/*.tar.gz
 
@@ -121,6 +126,7 @@ pushd $BASE_WORKSPACE
  fi
  if [ ! -f "$BASE_ZIP_IMG" ] || [ "$BASE_ZIP_IMG" == "" ]; then
  echo "Error: could not find image: $BASE_ZIP_IMG"
+ echo "On CustomPiOS v2 you can provide -d to download the latest image of your board automatically"
  exit 1
  fi
 
@@ -181,7 +187,7 @@ pushd $BASE_WORKSPACE
  CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
  MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
  MODULES_BEFORE="${MODULES}"
- ${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
+ ${CUSTOM_PI_OS_PATH}/custompios_core/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
  if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
  echo "Sourcing remote and submodules config"
  source "${REMOTE_AND_META_CONFIG}" ${@}

src/base_image_downloader.py → src/custompios_core/base_image_downloader.py

@@ -7,12 +7,13 @@
 import hashlib
 import shutil
 import re
+from typing import Dict, Any, Optional, cast, Tuple
+from common import get_image_config, read_images
 PRECENT_PROGRESS_SIZE = 5
 
 class ChecksumFailException(Exception):
  pass
 
-IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
 RETRY = 3
 
 def ensure_dir(d, chmod=0o777):
@@ -26,13 +27,31 @@ def ensure_dir(d, chmod=0o777):
  return False
  return True
 
-def read_images():
- if not os.path.isfile(IMAGES_CONFIG):
- raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
- with open(IMAGES_CONFIG,'r') as f:
- output = yaml.safe_load(f)
- return output
 
+def download_webpage(url: str) -> Optional[str]:
+ try:
+ with urllib.request.urlopen(url) as response:
+ # Decode the response to a string
+ webpage = response.read().decode('utf-8')
+ return webpage
+ except Exception as e:
+ print(str(e))
+ return None
+
+def get_location_header(url: str) -> str:
+ try:
+ with urllib.request.urlopen(url) as response:
+ response_url = response.url
+
+ if response_url is None:
+ raise Exception("Location header is None, can't determine latest rpi image")
+ return response_url
+ except Exception as e:
+ print(str(e))
+ print("Error: Failed to determine latest rpi image")
+ raise e
+
+
 class DownloadProgress:
  last_precent: float = 0
  def show_progress(self, block_num, block_size, total_size):
@@ -41,8 +60,10 @@ def show_progress(self, block_num, block_size, total_size):
  print(f"{new_precent}%", end="\r")
  self.last_precent = new_precent
 
-def get_file_name(headers):
- return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
+def get_file_name(headers, url):
+ if "Content-Disposition" in headers.keys():
+ return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
+ return url.split('/')[-1]
 
 def get_sha256(filename):
  sha256_hash = hashlib.sha256()
@@ -53,10 +74,12 @@ def get_sha256(filename):
  return file_checksum
  return
 
-def download_image_http(board: str, dest_folder: str, redownload: bool = False):
+def download_image_http(board: Dict[str, Any], dest_folder: str, redownload: bool = False):
  url = board["url"]
  checksum = board["checksum"]
+ download_http(url, checksum)
 
+def download_http(url: str, checksum_url: str, dest_folder: str, redownload: bool = False):
  with tempfile.TemporaryDirectory() as tmpdirname:
  print('created temporary directory', tmpdirname)
  temp_file_name = os.path.join(tmpdirname, "image.xz")
@@ -66,8 +89,8 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
  try:
  # Get sha and confirm its the right image
  download_progress = DownloadProgress()
- _, headers_checksum = urllib.request.urlretrieve(checksum, temp_file_checksum, download_progress.show_progress)
- file_name_checksum = get_file_name(headers_checksum)
+ _, headers_checksum = urllib.request.urlretrieve(checksum_url, temp_file_checksum, download_progress.show_progress)
+ file_name_checksum = get_file_name(headers_checksum, checksum_url)
 
  checksum_data = None
  with open(temp_file_checksum, 'r') as f:
@@ -82,13 +105,13 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
  if os.path.isfile(dest_file_name):
  file_checksum = get_sha256(dest_file_name)
  if file_checksum == online_checksum:
- # We got file and checksum is right
+ print("We got base image file and checksum is right")
  return
  # Get the file
  download_progress = DownloadProgress()
  _, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress)
 
- file_name = get_file_name(headers) 
+ file_name = get_file_name(headers, url)
  file_checksum = get_sha256(temp_file_name)
  if file_checksum != online_checksum:
  print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}')
@@ -102,11 +125,31 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
  else:
  print('Error encoutered at {RETRY} attempt')
  print(e)
+ exit(1)
  else:
  print(f"Success: {temp_file_name}")
  break
  return
 
+
+def download_image_rpi(board: Dict[str, Any], dest_folder: str):
+ port = board.get("port", "lite_armhf")
+ os_name = f"raspios"
+ distribution = board.get("distribution", "bookworm")
+ version_file = board.get("version_file", "latest")
+ version_folder = board.get("version_folder", "latest")
+
+ latest_url = f"https://downloads.raspberrypi.org/{os_name}_{port}_latest"
+
+ download_url = f"https://downloads.raspberrypi.org/{os_name}_{port}/images/{os_name}_{port}-{version_folder}/{version_file}-{os_name}-{distribution}-{port}.img.xz"
+ if version_file == "latest" or version_folder == "latest":
+ download_url = get_location_header(latest_url)
+
+ checksum_url = f"{download_url}.sha256"
+ download_http(download_url, checksum_url, dest_folder)
+ return
+
+
 if __name__ == "__main__":
  parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O')
  parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
@@ -118,14 +161,28 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
  base_board = os.environ.get("BASE_BOARD", None)
  base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
 
- if base_board is not None and base_board in images["images"]:
- if images["images"][base_board]["type"] == "http":
- download_image_http(images["images"][base_board], base_image_path)
- elif images["images"][base_board]["type"] == "torrent":
+ if base_image_path is None:
+ print(f'Error: did not find image config file')
+ exit(1)
+ cast(str, base_image_path)
+
+ image_config = get_image_config()
+ if image_config is not None:
+ if image_config["type"] == "http":
+ print(f"Downloading image for {base_board}")
+ download_image_http(image_config, base_image_path)
+ elif image_config["type"] == "rpi":
+ print(f"Downloading Raspberry Pi image for {base_board}")
+ download_image_rpi(image_config, base_image_path)
+ elif image_config["type"] == "torrent":
  print("Error: Torrent not implemented")
  exit(1)
  else:
- print("Error: Unsupported image download type")
+ print(f'Error: Unsupported image download type: {image_config["type"]}')
  exit(1)
+ else:
+ print(f"Error: Image config not found for: {base_board}")
+ exit(1)
+
 
- print("Done")
+ print("Done")

src/custompios_core/common.py

@@ -0,0 +1,32 @@
+""" Common functions between CustomPiOS python scripts"""
+from typing import Dict, Any, Optional, cast
+import yaml
+import os
+from pathlib import Path
+
+def get_custompios_folder():
+ custompios_path = os.environ.get("CUSTOM_PI_OS_PATH", None)
+ if custompios_path is not None:
+ return Path(custompios_path)
+ return Path(__file__).parent.parent
+
+
+IMAGES_CONFIG = os.path.join(get_custompios_folder(), "images.yml")
+
+
+def read_images() -> Dict[str, Dict[str,str]]:
+ if not os.path.isfile(IMAGES_CONFIG):
+ raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
+ with open(IMAGES_CONFIG,'r') as f:
+ output = yaml.safe_load(f)
+ return output
+
+def get_image_config() -> Optional[Dict["str", Any]]:
+ images = read_images()
+
+ base_board = os.environ.get("BASE_BOARD", None)
+ base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
+
+ if base_board is not None and base_board in images["images"]:
+ return images["images"][base_board]
+ return None

src/custompios_core/generate_board_config.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+import os
+import yaml
+from pathlib import Path
+from typing import Tuple, Optional, Dict, Any, cast
+import git
+from git import RemoteProgress
+from common import get_image_config
+import argparse
+import sys
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(add_help=True, description='Create an export shell script to use the yaml-configured variables')
+ parser.add_argument('output_script', type=str, help='path to output the chroot script master')
+ args = parser.parse_args()
+ image_config = get_image_config()
+ if image_config is None:
+ print("Error: Could not get image config")
+ sys.exit(1)
+ cast(Dict[str,Any], image_config)
+ if not "env" in image_config.keys():
+ print("Warning: no env in image config")
+ exit()
+ env = image_config["env"]
+ with open(args.output_script, "w+") as w:
+ for key in env.keys():
+ w.write(f'export {key}="{env[key]}"\n')
+

src/get_remote_modules.py → src/custompios_core/get_remote_modules.py

@@ -4,10 +4,11 @@
 from typing import Tuple, Optional
 import git
 from git import RemoteProgress
+from common import get_custompios_folder
 
 # TODO add env var to set this
-REMOTES_DIR = os.path.join(os.path.dirname(__file__), "remotes")
-REMOTE_CONFIG = os.path.join(os.path.dirname(__file__), "modules_remote.yml")
+REMOTES_DIR = os.path.join(get_custompios_folder(), "remotes")
+REMOTE_CONFIG = os.path.join(get_custompios_folder(), "modules_remote.yml")
 
 
 class CloneProgress(RemoteProgress):

src/custompios_core/list_boards.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python3
+from common import read_images
+
+if __name__ == "__main__":
+ images = read_images()["images"]
+ print("Available board targest for --board are:")
+ for key in sorted(images):
+ if "description" in images[key].keys():
+ print(f'{key} - {images[key]["description"]}')
+ else:
+ print(key)

src/custompios_core/multi_build.py

@@ -0,0 +1,15 @@
+import argparse
+
+def get_choices():
+ return ['rock', 'paper', 'scissors']
+
+def main():
+ parser = argparse.ArgumentParser(add_help=True, description='Build mulitple images for multiple devices')
+ parser.add_argument('--list', "-l", choices=get_choices(), type=str, nargs='+')
+ args = parser.parse_args()
+ print(args.list)
+ print("Done")
+ return
+
+if __name__ == "__main__":
+ main()