From c4373fa57a4aa9cd6c3dcf5cf0c38e0af5698a59 Mon Sep 17 00:00:00 2001
From: ColorfulRhino <131405023+ColorfulRhino@users.noreply.github.com>
Date: Fri, 14 Jun 2024 00:09:48 +0200
Subject: [PATCH] cli: Add command "dts-check"

Validates the dts/dtb file for the selected board and outputs the validation logs to the user.
This can be used when adding a new board, developing or improving a dts file. Should lead to higher quality device trees and patches overall, if used.
Will show warnings/errors if patches patch in some functionalities to a devicetree file without patching in the dt-bindings .yaml at the same time.
---
 lib/functions/cli/commands.sh                 |  6 +++++
 lib/functions/compilation/kernel-dts-check.sh | 22 +++++++++++++++++++
 lib/functions/compilation/kernel-make.sh      | 13 ++++++-----
 lib/functions/compilation/kernel.sh           | 13 +++++++++--
 lib/functions/general/python-tools.sh         | 17 +++++++++++---
 lib/library-functions.sh                      | 13 +++++++++--
 6 files changed, 71 insertions(+), 13 deletions(-)
 create mode 100644 lib/functions/compilation/kernel-dts-check.sh

diff --git a/lib/functions/cli/commands.sh b/lib/functions/cli/commands.sh
index 0430ec36c9e9..2b9207fb7d56 100644
--- a/lib/functions/cli/commands.sh
+++ b/lib/functions/cli/commands.sh
@@ -9,6 +9,8 @@
 
 function armbian_register_commands() {
 	# More than one command can map to the same handler. In that case, use ARMBIAN_COMMANDS_TO_VARS_DICT for specific vars.
+	# The handlers' functions "cli_${ARMBIAN_COMMAND_HANDLER}_pre_run" and "cli_${ARMBIAN_COMMAND_HANDLER}_run" get automatically called in "utils-cli.sh"
+	# Example: For command "docker-purge", the handler is "docker", which means the functions "cli_docker_pre_run" and "cli_docker_run" inside "cli-docker.sh are automatically called by "utils-cli.sh"
 	declare -g -A ARMBIAN_COMMANDS_TO_HANDLERS_DICT=(
 		["docker"]="docker" # thus requires cli_docker_pre_run and cli_docker_run
 		["docker-purge"]="docker"
@@ -70,6 +72,9 @@ function armbian_register_commands() {
 		["kernel-config"]="artifact"
 		["rewrite-kernel-config"]="artifact"
 
+		# Patch kernel and then check & validate the dtb file
+		["dts-check"]="artifact"				# Not really an artifact, but cli output only. Builds nothing.
+
 		["uboot"]="artifact"
 		["uboot-patch"]="artifact"
 		["atf-patch"]="artifact"
@@ -129,6 +134,7 @@ function armbian_register_commands() {
 		["rewrite-kernel-config"]="WHAT='kernel' KERNEL_CONFIGURE='yes' ARTIFACT_WILL_NOT_BUILD='yes' ARTIFACT_IGNORE_CACHE='yes' ${common_cli_artifact_vars}"
 		["kernel-patch"]="WHAT='kernel' CREATE_PATCHES='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
 		["kernel-dtb"]="WHAT='kernel' KERNEL_DTB_ONLY='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
+		["dts-check"]="WHAT='kernel' DTS_VALIDATE='yes' ARTIFACT_WILL_NOT_BUILD='yes'" # Not really an artifact, but cli output only. Builds nothing.
 
 		["uboot"]="WHAT='uboot' ${common_cli_artifact_vars}"
 		["uboot-config"]="WHAT='uboot' UBOOT_CONFIGURE='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
diff --git a/lib/functions/compilation/kernel-dts-check.sh b/lib/functions/compilation/kernel-dts-check.sh
new file mode 100644
index 000000000000..89b14bcc5a1b
--- /dev/null
+++ b/lib/functions/compilation/kernel-dts-check.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2013-2024 Armbian
+#
+# This file is a part of the Armbian Build Framework
+# https://github.com/armbian/build/
+
+
+# Validate the dts/dtb file against dt bindings found in "linux/Documentation/devicetree/bindings/"
+# See slide 15 in https://elinux.org/images/1/17/How_to_Get_Your_DT_Schema_Bindings_Accepted_in_Less_than_10_Iterations_-_Krzysztof_Kozlowski%2C_Linaro_-_ELCE_2023.pdf
+function validate_dts() {
+    [[ -z "${BOOT_FDT_FILE}" ]] && exit_with_error "BOOT_FDT_FILE not set! No dts file to validate."
+    display_alert "Validating dts/dtb file for selected board" "${BOOT_FDT_FILE} ; see output below" "info"
+
+    # "make CHECK_DTBS=y" uses the pip modules "dtschema" and "yamllint"
+    prepare_python_and_pip
+
+    # Run "make CHECK_DTBS=y" for the selected board's dtb file
+	run_kernel_make "CHECK_DTBS=y ${BOOT_FDT_FILE}"
+}
diff --git a/lib/functions/compilation/kernel-make.sh b/lib/functions/compilation/kernel-make.sh
index 5080a974f6c0..6b3cd5d16121 100644
--- a/lib/functions/compilation/kernel-make.sh
+++ b/lib/functions/compilation/kernel-make.sh
@@ -18,12 +18,13 @@ function run_kernel_make_internal() {
 	prepare_distcc_compilation_config
 
 	common_make_envs=(
-		"CCACHE_BASEDIR=\"$(pwd)\""              # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
-		"CCACHE_TEMPDIR=\"${CCACHE_TEMPDIR:?}\"" # Temporary directory for ccache, under WORKDIR
-		"PATH=\"${toolchain}:${PATH}\""          # Insert the toolchain first into the PATH.
-		"DPKG_COLORS=always"                     # Use colors for dpkg @TODO no dpkg is done anymore, remove?
-		"XZ_OPT='--threads=0'"                   # Use parallel XZ compression
-		"TERM='${TERM}'"                         # Pass the terminal type, so that 'make menuconfig' can work.
+		"CCACHE_BASEDIR=\"$(pwd)\""                                     # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
+		"CCACHE_TEMPDIR=\"${CCACHE_TEMPDIR:?}\""                        # Temporary directory for ccache, under WORKDIR
+		"PATH=\"${toolchain}:${PYTHON3_INFO[USERBASE]}/bin:${PATH}\""   # Insert the toolchain and the pip binaries into the PATH
+		"PYTHONPATH=\"${PYTHON3_INFO[MODULES_PATH]}:${PYTHONPATH}\""	# Insert the pip modules downloaded by Armbian into PYTHONPATH (needed for dtb checks)
+		"DPKG_COLORS=always"                                            # Use colors for dpkg @TODO no dpkg is done anymore, remove?
+		"XZ_OPT='--threads=0'"                                          # Use parallel XZ compression
+		"TERM='${TERM}'"                                                # Pass the terminal type, so that 'make menuconfig' can work.
 		"COLUMNS='${COLUMNS:-160}'"
 		"COLORFGBG='${COLORFGBG}'"
 	)
diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh
index a19a3d641db9..a6926e57f941 100644
--- a/lib/functions/compilation/kernel.sh
+++ b/lib/functions/compilation/kernel.sh
@@ -53,7 +53,7 @@ function compile_kernel() {
 	declare hash pre_patch_version
 	kernel_main_patching # has it's own logging sections inside
 
-	# Stop after patching;
+	# Stop after patching.
 	if [[ "${PATCH_ONLY}" == yes ]]; then
 		display_alert "PATCH_ONLY is set, stopping." "PATCH_ONLY=yes and patching success" "cachehit"
 		return 0
@@ -72,7 +72,6 @@ function compile_kernel() {
 	# re-read kernel version after patching
 	declare version
 	version=$(grab_version "$kernel_work_dir")
-	display_alert "Compiling $BRANCH kernel" "$version" "info"
 
 	# determine the toolchain
 	declare toolchain
@@ -80,6 +79,14 @@ function compile_kernel() {
 
 	kernel_config # has it's own logging sections inside
 
+	# Validate dts file if flag is set and stop after validation.
+	# Has to happen after kernel .config file was created
+	if [[ "${DTS_VALIDATE}" == yes ]]; then
+		LOG_SECTION="validate_dts" do_with_logging validate_dts
+		display_alert "DTS_VALIDATE is set, stopping." "DTS_VALIDATE=yes and dts sucessfully checked. See output above to fix your board's dts file." "cachehit"
+		return 0
+	fi
+
 	# Stop after configuring kernel, but only if using a specific CLI command ("kernel-config").
 	# Normal "KERNEL_CONFIGURE=yes" (during image build) is still allowed.
 	if [[ "${KERNEL_CONFIGURE}" == yes && "${ARMBIAN_COMMAND}" == *kernel-config ]]; then
@@ -87,6 +94,8 @@ function compile_kernel() {
 		return 0
 	fi
 
+	display_alert "Compiling $BRANCH kernel" "$version" "info"
+
 	# build via make and package .debs; they're separate sub-steps
 	kernel_prepare_build_and_package # has it's own logging sections inside
 
diff --git a/lib/functions/general/python-tools.sh b/lib/functions/general/python-tools.sh
index 9b49e2e6b137..7a357f937752 100644
--- a/lib/functions/general/python-tools.sh
+++ b/lib/functions/general/python-tools.sh
@@ -22,6 +22,8 @@ function early_prepare_pip3_dependencies_for_python_tools() {
 		"oras==0.1.30"        # for OCI stuff in mapper-oci-update
 		"Jinja2==3.1.4"       # for templating
 		"rich==13.7.1"        # for rich text formatting
+		"dtschema"            # for checking dts files and dt bindings (use latest version)
+		"yamllint"            # for checking dts files and dt bindings (use latest version)
 	)
 	return 0
 }
@@ -47,14 +49,20 @@ function prepare_python_and_pip() {
 	fi
 
 	# Check that the actual python3 --version is 3.9 at least
-	declare python3_version python3_full_version
-	python3_full_version="$("${python3_binary_path}" --version)" # "cut" below masks errors, do it twice.
+	declare python3_version python3_version_full
+	python3_version_full="$("${python3_binary_path}" --version)" # "cut" below masks errors, do it twice.
 	python3_version="$("${python3_binary_path}" --version | cut -d' ' -f2)"
-	display_alert "Python3 version" "${python3_version} - '${python3_full_version}'" "info"
+	display_alert "Python3 version" "${python3_version} - '${python3_version_full}'" "info"
 	if ! linux-version compare "${python3_version}" ge "3.9"; then
 		exit_with_error "Python3 version is too old (${python3_version}), need at least 3.9"
 	fi
 
+	declare python3_version_majorminor python3_version_string
+	# Extract the major and minor version numbers (e.g., "3.12" instead of "3.12.2")
+	python3_version_majorminor=$(echo "${python3_version_full}" | awk '{print $2}' | cut -d. -f1,2)
+	# Construct the version string (e.g., "python3.12")
+	python3_version_string="python$python3_version_majorminor"
+
 	# Check actual pip3 version
 	#   Note: we don't use "/usr/bin/pip3" at all, since it's commonly missing. instead "python -m pip"
 	#   The hostdep package python3-pip is still required, and other crazy might impact this.
@@ -100,16 +108,19 @@ function prepare_python_and_pip() {
 	declare python_hash_base="${python_pip_cache}/pip_pkg_hash"
 	declare python_hash_file="${python_hash_base}_${python3_pip_dependencies_hash}"
 	declare python3_user_base="${python_pip_cache}/base"
+	declare python3_modules_path="${python3_user_base}/lib/${python3_version_string}/site-packages"
 	declare python3_pycache="${python_pip_cache}/pycache"
 
 	# declare a readonly global dict with all needed info for executing stuff using this setup
 	declare -r -g -A PYTHON3_INFO=(
 		[BIN]="${python3_binary_path}"
 		[USERBASE]="${python3_user_base}"
+		[MODULES_PATH]="${python3_modules_path}"
 		[PYCACHEPREFIX]="${python3_pycache}"
 		[HASH]="${python3_pip_dependencies_hash}"
 		[DEPS]="${python3_pip_dependencies[*]}"
 		[VERSION]="${python3_version}"
+		[VERSION_STRING]="${python3_version_string}"
 		[PIP_VERSION]="${pip3_version}"
 	)
 
diff --git a/lib/library-functions.sh b/lib/library-functions.sh
index f970001fa25b..da459ea79d62 100644
--- a/lib/library-functions.sh
+++ b/lib/library-functions.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# This file is/was autogenerated by lib/tools/gen-library.sh; don't modify manually
+# This file is/was autogenerated by ./lib/tools/gen-library.sh; don't modify manually
 
 # no errors tolerated. invoked before each sourced file to make sure.
 #set -o pipefail  # trace ERR through pipes - will be enabled "soon"
@@ -361,6 +361,15 @@ set -o errexit  ## set -e : exit the script if any statement returns a non-true
 # shellcheck source=lib/functions/compilation/kernel-debs.sh
 source "${SRC}"/lib/functions/compilation/kernel-debs.sh
 
+# no errors tolerated. invoked before each sourced file to make sure.
+#set -o pipefail  # trace ERR through pipes - will be enabled "soon"
+#set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
+set -o errtrace # trace ERR through - enabled
+set -o errexit  ## set -e : exit the script if any statement returns a non-true return value - enabled
+### lib/functions/compilation/kernel-dts-check.sh
+# shellcheck source=lib/functions/compilation/kernel-dts-check.sh
+source "${SRC}"/lib/functions/compilation/kernel-dts-check.sh
+
 # no errors tolerated. invoked before each sourced file to make sure.
 #set -o pipefail  # trace ERR through pipes - will be enabled "soon"
 #set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
@@ -1185,4 +1194,4 @@ source "${SRC}"/lib/functions/rootfs/trap-rootfs.sh
 #set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
 set -o errtrace # trace ERR through - enabled
 set -o errexit  ## set -e : exit the script if any statement returns a non-true return value - enabled
-# This file is/was autogenerated by lib/tools/gen-library.sh; don't modify manually
+# This file is/was autogenerated by ./lib/tools/gen-library.sh; don't modify manually