cli: Add command "dts-check" to validate dts files and improve board & patch development overall (resubmission)

.github/dependabot.yml

@@ -4,3 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"

lib/functions/cli/commands.sh

@@ -9,6 +9,8 @@
 
 function armbian_register_commands() {
 	# More than one command can map to the same handler. In that case, use ARMBIAN_COMMANDS_TO_VARS_DICT for specific vars.
+	# The handlers' functions "cli_${ARMBIAN_COMMAND_HANDLER}_pre_run" and "cli_${ARMBIAN_COMMAND_HANDLER}_run" get automatically called in "utils-cli.sh"
+	# Example: For command "docker-purge", the handler is "docker", which means the functions "cli_docker_pre_run" and "cli_docker_run" inside "cli-docker.sh are automatically called by "utils-cli.sh"
 	declare -g -A ARMBIAN_COMMANDS_TO_HANDLERS_DICT=(
 		["docker"]="docker" # thus requires cli_docker_pre_run and cli_docker_run
 		["docker-purge"]="docker"
@@ -70,6 +72,9 @@ function armbian_register_commands() {
 		["kernel-config"]="artifact"
 		["rewrite-kernel-config"]="artifact"
 
+		# Patch kernel and then check & validate the dtb file
+		["dts-check"]="artifact"				# Not really an artifact, but cli output only. Builds nothing.
+
 		["uboot"]="artifact"
 		["uboot-patch"]="artifact"
 		["atf-patch"]="artifact"
@@ -129,6 +134,7 @@ function armbian_register_commands() {
 		["rewrite-kernel-config"]="WHAT='kernel' KERNEL_CONFIGURE='yes' ARTIFACT_WILL_NOT_BUILD='yes' ARTIFACT_IGNORE_CACHE='yes' ${common_cli_artifact_vars}"
 		["kernel-patch"]="WHAT='kernel' CREATE_PATCHES='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
 		["kernel-dtb"]="WHAT='kernel' KERNEL_DTB_ONLY='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
+		["dts-check"]="WHAT='kernel' DTS_VALIDATE='yes' ARTIFACT_WILL_NOT_BUILD='yes'" # Not really an artifact, but cli output only. Builds nothing.
 
 		["uboot"]="WHAT='uboot' ${common_cli_artifact_vars}"
 		["uboot-config"]="WHAT='uboot' UBOOT_CONFIGURE='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"

lib/functions/compilation/kernel-dts-check.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2013-2024 Armbian
+#
+# This file is a part of the Armbian Build Framework
+# https://github.com/armbian/build/
+
+
+# Validate the dts/dtb file against dt bindings found in "linux/Documentation/devicetree/bindings/"
+# See slide 15 in https://elinux.org/images/1/17/How_to_Get_Your_DT_Schema_Bindings_Accepted_in_Less_than_10_Iterations_-_Krzysztof_Kozlowski%2C_Linaro_-_ELCE_2023.pdf
+function validate_dts() {
+    [[ -z "${BOOT_FDT_FILE}" ]] && exit_with_error "BOOT_FDT_FILE not set! No dts file to validate."
+    display_alert "Validating dts/dtb file for selected board" "${BOOT_FDT_FILE} ; see output below" "info"
+
+    # "make CHECK_DTBS=y" uses the pip modules "dtschema" and "yamllint"
+    prepare_python_and_pip
+
+    # Run "make CHECK_DTBS=y" for the selected board's dtb file
+	run_kernel_make "CHECK_DTBS=y ${BOOT_FDT_FILE}"
+}

lib/functions/compilation/kernel-make.sh

@@ -18,12 +18,13 @@ function run_kernel_make_internal() {
 	prepare_distcc_compilation_config
 
 	common_make_envs=(
-		"CCACHE_BASEDIR=\"$(pwd)\""              # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
-		"CCACHE_TEMPDIR=\"${CCACHE_TEMPDIR:?}\"" # Temporary directory for ccache, under WORKDIR
-		"PATH=\"${toolchain}:${PATH}\""          # Insert the toolchain first into the PATH.
-		"DPKG_COLORS=always"                     # Use colors for dpkg @TODO no dpkg is done anymore, remove?
-		"XZ_OPT='--threads=0'"                   # Use parallel XZ compression
-		"TERM='${TERM}'"                         # Pass the terminal type, so that 'make menuconfig' can work.
+		"CCACHE_BASEDIR=\"$(pwd)\""                                     # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
+		"CCACHE_TEMPDIR=\"${CCACHE_TEMPDIR:?}\""                        # Temporary directory for ccache, under WORKDIR
+		"PATH=\"${toolchain}:${PYTHON3_INFO[USERBASE]}/bin:${PATH}\""   # Insert the toolchain and the pip binaries into the PATH
+		"PYTHONPATH=\"${PYTHON3_INFO[MODULES_PATH]}:${PYTHONPATH}\""	# Insert the pip modules downloaded by Armbian into PYTHONPATH (needed for dtb checks)
+		"DPKG_COLORS=always"                                            # Use colors for dpkg @TODO no dpkg is done anymore, remove?
+		"XZ_OPT='--threads=0'"                                          # Use parallel XZ compression
+		"TERM='${TERM}'"                                                # Pass the terminal type, so that 'make menuconfig' can work.
 		"COLUMNS='${COLUMNS:-160}'"
 		"COLORFGBG='${COLORFGBG}'"
 	)

lib/functions/compilation/kernel.sh

@@ -53,7 +53,7 @@ function compile_kernel() {
 	declare hash pre_patch_version
 	kernel_main_patching # has it's own logging sections inside
 
-	# Stop after patching;
+	# Stop after patching.
 	if [[ "${PATCH_ONLY}" == yes ]]; then
 		display_alert "PATCH_ONLY is set, stopping." "PATCH_ONLY=yes and patching success" "cachehit"
 		return 0
@@ -72,21 +72,30 @@ function compile_kernel() {
 	# re-read kernel version after patching
 	declare version
 	version=$(grab_version "$kernel_work_dir")
-	display_alert "Compiling $BRANCH kernel" "$version" "info"
 
 	# determine the toolchain
 	declare toolchain
 	LOG_SECTION="kernel_determine_toolchain" do_with_logging do_with_hooks kernel_determine_toolchain
 
 	kernel_config # has it's own logging sections inside
 
+	# Validate dts file if flag is set and stop after validation.
+	# Has to happen after kernel .config file was created
+	if [[ "${DTS_VALIDATE}" == yes ]]; then
+		LOG_SECTION="validate_dts" do_with_logging validate_dts
+		display_alert "DTS_VALIDATE is set, stopping." "DTS_VALIDATE=yes and dts sucessfully checked. See output above to fix your board's dts file." "cachehit"
+		return 0
+	fi
+
 	# Stop after configuring kernel, but only if using a specific CLI command ("kernel-config").
 	# Normal "KERNEL_CONFIGURE=yes" (during image build) is still allowed.
 	if [[ "${KERNEL_CONFIGURE}" == yes && "${ARMBIAN_COMMAND}" == *kernel-config ]]; then
 		display_alert "Stopping after configuring kernel" "" "cachehit"
 		return 0
 	fi
 
+	display_alert "Compiling $BRANCH kernel" "$version" "info"
+
 	# build via make and package .debs; they're separate sub-steps
 	kernel_prepare_build_and_package # has it's own logging sections inside
 

lib/functions/general/python-tools.sh

@@ -11,21 +11,6 @@
 # If you know to tame it, teach me. I'd rather not know about PYTHONUSERBASE and such.
 # --rpardini
 
-function early_prepare_pip3_dependencies_for_python_tools() {
-	# This is like a stupid version of requirements.txt
-	declare -a -g python3_pip_dependencies=(
-		"unidiff==0.7.4"      # for parsing unified diff
-		"GitPython==3.1.30"   # for manipulating git repos
-		"unidecode==1.3.6"    # for converting strings to ascii
-		"coloredlogs==15.0.1" # for colored logging
-		"PyYAML==6.0.1"       # for parsing/writing YAML
-		"oras==0.1.17"        # for OCI stuff in mapper-oci-update
-		"Jinja2==3.1.2"       # for templating
-		"rich==13.4.1"        # for rich text formatting
-	)
-	return 0
-}
-
 # call: prepare_python_and_pip # this defines global PYTHON3_INFO dict and PYTHON3_VARS array
 function prepare_python_and_pip() {
 	assert_prepared_host # this needs a prepared host to work; avoid fake errors about "python3-pip" not being installed
@@ -47,14 +32,20 @@ function prepare_python_and_pip() {
 	fi
 
 	# Check that the actual python3 --version is 3.9 at least
-	declare python3_version python3_full_version
-	python3_full_version="$("${python3_binary_path}" --version)" # "cut" below masks errors, do it twice.
+	declare python3_version python3_version_full
+	python3_version_full="$("${python3_binary_path}" --version)" # "cut" below masks errors, do it twice.
 	python3_version="$("${python3_binary_path}" --version | cut -d' ' -f2)"
-	display_alert "Python3 version" "${python3_version} - '${python3_full_version}'" "info"
+	display_alert "Python3 version" "${python3_version} - '${python3_version_full}'" "info"
 	if ! linux-version compare "${python3_version}" ge "3.9"; then
 		exit_with_error "Python3 version is too old (${python3_version}), need at least 3.9"
 	fi
 
+	declare python3_version_majorminor python3_version_string
+	# Extract the major and minor version numbers (e.g., "3.12" instead of "3.12.2")
+	python3_version_majorminor=$(echo "${python3_version_full}" | awk '{print $2}' | cut -d. -f1,2)
+	# Construct the version string (e.g., "python3.12")
+	python3_version_string="python$python3_version_majorminor"
+
 	# Check actual pip3 version
 	#   Note: we don't use "/usr/bin/pip3" at all, since it's commonly missing. instead "python -m pip"
 	#   The hostdep package python3-pip is still required, and other crazy might impact this.
@@ -68,9 +59,15 @@ function prepare_python_and_pip() {
 	display_alert "pip3 version" "${pip3_version_number}: '${pip3_version}'" "info"
 
 	# Hash the contents of the dependencies array + the Python version + the release
+	declare python3_pip_dependencies_path
 	declare python3_pip_dependencies_hash
-	early_prepare_pip3_dependencies_for_python_tools
-	python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${pip3_version}" "${python3_pip_dependencies[*]}" | sha256sum | cut -d' ' -f1)"
+
+	python3_pip_dependencies_path="${SRC}/requirements.txt"
+	# Check for the existence of requirements.txt, fail if not found
+	[[ ! -f "${python3_pip_dependencies_path}" ]] && exit_with_error "Python Pip requirements.txt file not found at path: ${python3_pip_dependencies_path}"
+
+	# Calculate the hash for the Pip requirements
+	python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${pip3_version}" "$(cat "${python3_pip_dependencies_path}")" | sha256sum | cut -d' ' -f1)"
 
 	declare non_cache_dir="/armbian-pip"
 	declare python_pip_cache="${SRC}/cache/pip"
@@ -100,16 +97,19 @@ function prepare_python_and_pip() {
 	declare python_hash_base="${python_pip_cache}/pip_pkg_hash"
 	declare python_hash_file="${python_hash_base}_${python3_pip_dependencies_hash}"
 	declare python3_user_base="${python_pip_cache}/base"
+	declare python3_modules_path="${python3_user_base}/lib/${python3_version_string}/site-packages"
 	declare python3_pycache="${python_pip_cache}/pycache"
 
 	# declare a readonly global dict with all needed info for executing stuff using this setup
 	declare -r -g -A PYTHON3_INFO=(
 		[BIN]="${python3_binary_path}"
 		[USERBASE]="${python3_user_base}"
+		[MODULES_PATH]="${python3_modules_path}"
 		[PYCACHEPREFIX]="${python3_pycache}"
-		[HASH]="${python3_pip_dependencies_hash}"
-		[DEPS]="${python3_pip_dependencies[*]}"
+		[REQUIREMENTS_HASH]="${python3_pip_dependencies_hash}"
+		[REQUIREMENTS_PATH]="${python3_pip_dependencies_path}"
 		[VERSION]="${python3_version}"
+		[VERSION_STRING]="${python3_version_string}"
 		[PIP_VERSION]="${pip3_version}"
 	)
 
@@ -128,7 +128,7 @@ function prepare_python_and_pip() {
 		# remove the old hashes matching base, don't leave junk behind
 		run_host_command_logged rm -fv "${python_hash_base}*"
 
-		run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" -m pip install "${pip3_extra_args[@]}" "${python3_pip_dependencies[@]}"
+		run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" -m pip install "${pip3_extra_args[@]}" -r "${python3_pip_dependencies_path}"
 
 		# Create the hash file
 		run_host_command_logged touch "${python_hash_file}"

lib/library-functions.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# This file is/was autogenerated by lib/tools/gen-library.sh; don't modify manually
+# This file is/was autogenerated by ./lib/tools/gen-library.sh; don't modify manually
 
 # no errors tolerated. invoked before each sourced file to make sure.
 #set -o pipefail  # trace ERR through pipes - will be enabled "soon"
@@ -361,6 +361,15 @@ set -o errexit  ## set -e : exit the script if any statement returns a non-true
 # shellcheck source=lib/functions/compilation/kernel-debs.sh
 source "${SRC}"/lib/functions/compilation/kernel-debs.sh
 
+# no errors tolerated. invoked before each sourced file to make sure.
+#set -o pipefail  # trace ERR through pipes - will be enabled "soon"
+#set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
+set -o errtrace # trace ERR through - enabled
+set -o errexit  ## set -e : exit the script if any statement returns a non-true return value - enabled
+### lib/functions/compilation/kernel-dts-check.sh
+# shellcheck source=lib/functions/compilation/kernel-dts-check.sh
+source "${SRC}"/lib/functions/compilation/kernel-dts-check.sh
+
 # no errors tolerated. invoked before each sourced file to make sure.
 #set -o pipefail  # trace ERR through pipes - will be enabled "soon"
 #set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
@@ -1185,4 +1194,4 @@ source "${SRC}"/lib/functions/rootfs/trap-rootfs.sh
 #set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
 set -o errtrace # trace ERR through - enabled
 set -o errexit  ## set -e : exit the script if any statement returns a non-true return value - enabled
-# This file is/was autogenerated by lib/tools/gen-library.sh; don't modify manually
+# This file is/was autogenerated by ./lib/tools/gen-library.sh; don't modify manually

requirements.txt

@@ -0,0 +1,17 @@
+# Requirements/dependencies for various Python scripts used in the Armbian Build Framework
+#
+# IMPORTANT NOTES:
+# When adding a new requirements, please leave a comment to explain its purpose.
+# Always use a fixed version, this is important for correct hashing.
+# Dependabot will keep these versions up to date.
+
+unidiff == 0.7.5       # for parsing unified diff
+GitPython == 3.1.43    # for manipulating git repos
+unidecode == 1.3.8     # for converting strings to ascii
+coloredlogs == 15.0.1  # for colored logging
+PyYAML == 6.0.1        # for parsing/writing YAML
+oras == 0.1.30         # for OCI stuff in mapper-oci-update
+Jinja2 == 3.1.4        # for templating
+rich == 13.7.1         # for rich text formatting
+dtschema == 2024.5     # for checking dts files and dt bindings
+yamllint == 1.35.1     # for checking dts files and dt bindings