From 1fcb0f06969ac0ba1d77fd7fdd9c6361947f9828 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Thu, 4 Apr 2024 10:03:41 +0200 Subject: [PATCH 1/6] pkg/mkrootfs-ext4: add extra 200MB for the resulting rootfs We need this extra space for updating packages while building live image: make LIVE_PKG=

live. The whole implementation follows. Stay tuned. Signed-off-by: Roman Penyaev --- pkg/mkrootfs-ext4/make-rootfs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/mkrootfs-ext4/make-rootfs b/pkg/mkrootfs-ext4/make-rootfs index cde4d5a155..5d2824b792 100755 --- a/pkg/mkrootfs-ext4/make-rootfs +++ b/pkg/mkrootfs-ext4/make-rootfs @@ -41,6 +41,9 @@ IMGFILE=/rootfs.img ROOTFS_FILE_SIZE_KB=$(du -sk . | awk '{print $1}') ROOTFS_PART_BLOCKS=$(( $ROOTFS_FILE_SIZE_KB / 4 + $ROOTFS_PART_HEADROOM_BLOCKS )) ROOTFS_PART_SIZE=$(( $ROOTFS_PART_BLOCKS * $ROOTFS_BLOCKSZ )) + # Increase fs size on 200MB. We need extra room for updating packages while + # building live image. See 'LIVE_PKG=

live' in the Makefile. + ROOTFS_PART_SIZE=$((ROOTFS_PART_SIZE + (200<<20))) ROOTFS_PART_SIZE_KB=$(((ROOTFS_PART_SIZE + 1023) / 1024)) dd if=/dev/zero of=$IMGFILE bs=1024 seek="$ROOTFS_PART_SIZE_KB" count=0 From 5eb36d697713defb962ee8d2ee8dbceef4763bc9 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Thu, 4 Apr 2024 10:05:19 +0200 Subject: [PATCH 2/6] tools/parse-pkgs: introduce -p parameter By -p parameter the resulting yml will be reduced to exactly the package specified, for example if input contains kernel, init, onboot and services sections and "pillar" is specified as a package, then the resulting yml file will contain only elements and sections which mention "pillar", and not other packages. Examples of reduced output based on rootfs-kvm-generic.yml.in input: $ ./tools/parse-pkgs.sh -p pkg/grub images/out/rootfs-kvm-generic.yml.in init: - lfedge/eve-grub:d16eeeb340b3f2ff8cb6e60be65c586e72de9aae-amd64 $ ./tools/parse-pkgs.sh -p pkg/kernel images/out/rootfs-kvm-generic.yml.in kernel: image: docker.io/lfedge/eve-kernel:eve-kernel-amd64-v6.1.38-generic-ae347d3a26ec-gcc cmdline: "rootdelay=3" $ ./tools/parse-pkgs.sh -p pkg/pillar images/out/rootfs-kvm-generic.yml.in services: - name: pillar image: lfedge/eve-pillar:39a33fb49dfb6c17ee8b373dd4f934f151c25136-amd64 cgroupsPath: /eve/services/pillar oomScoreAdj: -999 onboot: # If you change the order of pillar-onboot don't forget to # change /containers/onboot/005-pillar-onboot/lower in pkg/mkimage-raw-efi accordingly: # 005-pillar-onboot must follow the order number of pillar-onboot # onboot part of pillar to prepare services to start - name: pillar-onboot image: lfedge/eve-pillar:39a33fb49dfb6c17ee8b373dd4f934f151c25136-amd64 command: ["/opt/zededa/bin/onboot.sh"] Signed-off-by: Roman Penyaev --- tools/parse-pkgs.sh | 55 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tools/parse-pkgs.sh b/tools/parse-pkgs.sh index bb3ffbae1c..a055b9e748 100755 --- a/tools/parse-pkgs.sh +++ b/tools/parse-pkgs.sh @@ -125,6 +125,27 @@ KUBE_TAG=${KUBE_TAG} EOF } +keep_only_pkg() { + local pkg + pkg=$1 + + # Don't ask me how the stuff below works! Don't even try to reach me out. + # I swear I do not know. Everything copied from Stackoverflow and after + # a bunch of iteration I got what I wanted. Ok, Ok. What I do know is + # why we need sed at the end of the pipe. It's easy. We remove "- null" + # elements from array left there by 'yq' transform, which I do not know + # how to remove using yq. + if [ "$pkg" == "kernel" ]; then + cat | yq ".kernel.[] as \$i ireduce({}; setpath(\$i | path; \$i))" | + sed '/^\s\+-\s\+null\s*$/d' + else + cat | yq "( (.init.[] | select(. == \"lfedge/eve-$pkg:*\")), + ((.services, .onboot).[] | select(.image == \"lfedge/eve-$pkg:*\")) ) as + \$i ireduce({}; setpath(\$i | path; \$i))" | + sed '/^\s\+-\s\+null\s*$/d' + fi +} + if [ -z "$DOCKER_ARCH_TAG" ] ; then case $(uname -m) in x86_64) ARCH=-amd64 @@ -182,9 +203,41 @@ KUBE_TAG=$(linuxkit_tag pkg/kube) # on the previous tags being already defined. EVE_TAG=$(synthetic_tag zededa/eve pkg/eve/Dockerfile.in) +help() { + echo "Usage: $0 [ [-p ]]" >&2 + echo + echo " substitutes *_TAG in the input file and outputs " >&2 + echo " results to stdout." >&2 + echo + echo " -p pkg - Removes everything which is not the from " >&2 + echo " the resulting output." >&2 + echo + exit 1 +} + +unset pkg +while getopts "p:h" o +do + case $o in + p) + pkg=$(basename "$OPTARG") + ;; + h) + help + ;; + *) + ;; + esac +done +shift $((OPTIND-1)) + TAGS=$(gen_tags) if [ $# -ge 1 ]; then - resolve_tags "$TAGS" "$1" + out=$(resolve_tags "$TAGS" "$1") + if [ -n "$pkg" ]; then + out=$(echo "$out" | keep_only_pkg "$pkg") + fi + echo "$out" else echo "$TAGS" fi From 718fb2e2d2afc4bbd4437260ae0fd3809eb6965c Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Thu, 4 Apr 2024 10:19:23 +0200 Subject: [PATCH 3/6] Makefile: don't regenerate yml* targets if RESCAN_DEPS is not FORCE Targets `images/out` and `build-tools` should be put as order-only-prerequisites: https://www.gnu.org/savannah-checkouts/gnu/make/manual/html_node/Prerequisite-Types.html#Prerequisite-Types Because `images/out` is a folder, which timestamp is always changes if new files are modified inside and `build-tools` is .PHONY target. This patch moves `images/out` and `build-tools` to the order-only-prerequisites to avoid constant yml* regeneration, which forces rebuild images. The force of rebuilding is controlled by the RESCAN_DEPS variable, which is set to FORCE (is .PHONY target), which means the default behavior won't break, but gives us freedom to reset the RESCAN_DEPS and do not generate live image if it was already generated. This 'do-not-generate' live image logic will be used in the next patch for the 'LIVE_PKG' update. Signed-off-by: Roman Penyaev --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a885093ad0..cb360a0af7 100644 --- a/Makefile +++ b/Makefile @@ -942,7 +942,7 @@ endif qemu-img resize $@ ${MEDIA_SIZE}M $(QUIET): $@: Succeeded -%.yml: %.yml.in build-tools $(RESCAN_DEPS) +%.yml: %.yml.in $(RESCAN_DEPS) | build-tools $(QUIET)$(PARSE_PKGS) $< > $@ $(QUIET): $@: Succeeded @@ -977,7 +977,7 @@ eve-%: pkg/%/Dockerfile build-tools $(RESCAN_DEPS) images/out: mkdir -p $@ -images/out/rootfs-%.yml.in: images/rootfs.yml.in images/out FORCE +images/out/rootfs-%.yml.in: images/rootfs.yml.in $(RESCAN_DEPS) | images/out $(QUIET)tools/compose-image-yml.sh -b $< -v "$(ROOTFS_VERSION)-$*-$(ZARCH)" -o $@ -h $(HV) $(patsubst %,images/modifiers/%.yq,$(subst -, ,$*)) test-images-patches: $(patsubst images/modifiers/%.yq,images/out/rootfs-%.yml.in,$(wildcard images/modifiers/*.yq)) From 8172e93f4f52f831dd8505765e0cf808e6196690 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Thu, 4 Apr 2024 10:25:49 +0200 Subject: [PATCH 4/6] Makefile: introduce 'make LIVE_PKG= live' mode The main problem with the EVE build system (with its other features) is its complete inability to detect changes and create images from what has actually been changed, which leads to complete rebuild of the live image every time you made a small change and takes: $ time make live real 3m15.571s user 1m7.485s sys 0m5.301s So much waste of time for daily developer work. This can be slightly improved by generating the ext4 rootfs image: $ time make ROOTFS_FORMAT=ext4 live real 2m8.087s user 1m45.201s sys 0m7.096s Yeah, gets better now. But not ideal. For daily developer routine we don't need to rebuild the whole image if you've modified only one component, which is usually the case. Let's say your daily work is to tweak pillar and live qcow2 image has been already generated with ext4 rootfs, so why the image has to be regenerated each time you did a change in pillar? No reason at all. This change enables the LIVE_PKG= mode for qcow2 live image generations (with ext4 as rootfs, this is the requirement). This mode does not generates the whole qcow2 image from the scratch, but mounts rootfs and copies prepared tarball. Easy. To make things even faster prepared tarball contains only the required package, not the whole rootfs. This is made by the reducing rootfs yml (see previous patches). What it gives us? Not speed of light, unfortunately, but tens of seconds, look at the numbers: time make LIVE_PKG=pkg/dom0-ztools live real 0m9.387s user 0m4.919s sys 0m3.216s time make LIVE_PKG=pkg/grub live real 0m7.059s user 0m3.050s sys 0m2.307s time make LIVE_PKG=pkg/pillar live real 0m22.470s user 0m19.443s sys 0m3.696s (yeah, pillar is a fat guy) Limitations? Yes, a couple: 1. live qcow2 should be generated once with the ROOTFS_FORMAT=ext4 2. Don't forget what package you've changed, otherwise your image will be patched with something different. 3. Probably something does not work, but will be found and fixed if more people use that. 4. Probably something is broken. Same, will be fixed. Signed-off-by: Roman Penyaev --- Makefile | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb360a0af7..f2f524b01f 100644 --- a/Makefile +++ b/Makefile @@ -40,8 +40,13 @@ DEV=n MEDIA_SIZE=32768 # Image type for final disk images IMG_FORMAT=qcow2 +ifdef LIVE_PKG +# For live pkg updates we support read-write FS, like ext4 +ROOTFS_FORMAT=ext4 +else # Filesystem type for rootfs image ROOTFS_FORMAT?=squash +endif # Image type for installer image INSTALLER_IMG_FORMAT=raw # Image type for verification image @@ -156,6 +161,7 @@ ROOTFS_COMPLETE=$(ROOTFS_FULL_NAME)-%-$(ZARCH).$(ROOTFS_FORMAT) ROOTFS_IMG=$(ROOTFS).img # ROOTFS_TAR is in BUILD_DIR, not installer, so it does not get installed ROOTFS_TAR=$(BUILD_DIR)/rootfs.tar +ROOTFS_REDUCED_TAR=$(BUILD_DIR)/rootfs.reduced.tar CONFIG_IMG=$(INSTALLER)/config.img INITRD_IMG=$(INSTALLER)/initrd.img INSTALLER_IMG=$(INSTALLER)/installer.img @@ -300,7 +306,12 @@ LINUXKIT_SOURCE=https://github.com/linuxkit/linuxkit.git LINUXKIT_OPTS=$(if $(strip $(EVE_HASH)),--hash) $(EVE_HASH) $(if $(strip $(EVE_REL)),--release) $(EVE_REL) LINUXKIT_PKG_TARGET=build LINUXKIT_PATCHES_DIR=tools/linuxkit/patches +ifdef LIVE_PKG +# Don't regenerate everything once generated +RESCAN_DEPS= +else RESCAN_DEPS=FORCE +endif # set FORCE_BUILD to --force to enforce rebuild FORCE_BUILD= @@ -611,7 +622,14 @@ rootfs: $(ROOTFS_TAR) $(ROOTFS_IMG) current rootfs.tar: $(ROOTFS_TAR) rootfstar: $(ROOTFS_TAR) sbom: $(SBOM) -live: $(LIVE_IMG) $(BIOS_IMG) current ; $(QUIET): "$@: Succeeded, LIVE_IMG=$(LIVE_IMG)" +ifdef LIVE_PKG +live: $(LIVE_IMG) $(BIOS_IMG) current $(ROOTFS_REDUCED_TAR) +# Mount first rootfs and copy all files to / from reduced tar. One line magic. + guestfish -a $(BUILD_DIR)/live.qcow2 run : mount /dev/sda2 / : tar-in $(ROOTFS_REDUCED_TAR) / + $(QUIET): $@: Succeeded +else +live: $(LIVE_IMG) $(BIOS_IMG) current ; $(QUIET): "$@: Succeeded, LIVE_IMG=$(LIVE_IMG)" +endif live-%: $(LIVE).% ; $(QUIET): "$@: Succeeded, LIVE=$(LIVE)" installer: $(INSTALLER_IMG) installer-%: $(INSTALLER).% current ; @echo "$@: Succeeded, INSTALLER_IMG=$<" @@ -638,6 +656,11 @@ $(ROOTFS)-%.img: $(ROOTFS_IMG) @rm -f $@ && ln -s $(notdir $<) $@ $(QUIET): $@: Succeeded +$(ROOTFS_REDUCED_TAR): images/out/rootfs-$(HV)-$(PLATFORM).yml.reduced + $(QUIET): $@: Begin + ./tools/makerootfs.sh tar -y $< -t $@ -d $(INSTALLER) -a $(ZARCH) + $(QUIET): $@: Succeeded + $(ROOTFS_TAR): images/out/rootfs-$(HV)-$(PLATFORM).yml | $(INSTALLER) $(QUIET): $@: Begin ./tools/makerootfs.sh tar -y $< -t $@ -d $(INSTALLER) -a $(ZARCH) @@ -942,6 +965,9 @@ endif qemu-img resize $@ ${MEDIA_SIZE}M $(QUIET): $@: Succeeded +%.yml.reduced: %.yml.in FORCE | build-tools + $(QUIET)$(PARSE_PKGS) -p $(LIVE_PKG) $< > $@ + %.yml: %.yml.in $(RESCAN_DEPS) | build-tools $(QUIET)$(PARSE_PKGS) $< > $@ $(QUIET): $@: Succeeded From c163ae3e0a0e2fe77ca59f2c9bd74d68d72348ff Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Thu, 4 Apr 2024 09:53:27 +0200 Subject: [PATCH 5/6] Makefile: update usage with LIVE_PKG description make LIVE_PKG= live updates existing qcow2 disk image of EVE with RW rootfs (ext4) by only copying specified package. This significantly reduced overall build time of the disk image. Used by developers only! Signed-off-by: Roman Penyaev --- Makefile | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index f2f524b01f..2bd085bfd4 100644 --- a/Makefile +++ b/Makefile @@ -1060,19 +1060,21 @@ help: @echo " bump-eve-api bump eve-api in all subprojects" @echo @echo "Commonly used build targets:" - @echo " build-tools builds linuxkit utilities and installs under build-tools/bin" - @echo " config builds a bundle with initial EVE configs" - @echo " pkgs builds all EVE packages" - @echo " pkg/XXX builds XXX EVE package" - @echo " rootfs builds default EVE rootfs image (upload it to the cloud as BaseImage)" - @echo " live builds a full disk image of EVE which can be function as a virtual device" - @echo " live-XXX builds a particular kind of EVE live image (raw, qcow2, gcp, vdi, parallels)" - @echo " installer-raw builds raw disk installer image (to be installed on bootable media)" - @echo " installer-iso builds an ISO installers image (to be installed on bootable media)" - @echo " installer-net builds a tarball of artifacts to be used for PXE booting" - @echo " verification-raw builds raw disk verification image (to be installed on bootable media)" - @echo " verification-net builds a tarball of artifacts to be used for PXE verification" - @echo " verification-iso builds an ISO verification image (to be installed on bootable media)" + @echo " build-tools builds linuxkit utilities and installs under build-tools/bin" + @echo " config builds a bundle with initial EVE configs" + @echo " pkgs builds all EVE packages" + @echo " pkg/XXX builds XXX EVE package" + @echo " rootfs builds default EVE rootfs image (upload it to the cloud as BaseImage)" + @echo " live builds a full disk image of EVE which can be function as a virtual device" + @echo " LIVE_PKG= live updates existing qcow2 disk image of EVE with RW rootfs (ext4) by only copying specified package. This significantly reduced overall build time of the disk image. Used by developers only!" + + @echo " live-XXX builds a particular kind of EVE live image (raw, qcow2, gcp, vdi, parallels)" + @echo " installer-raw builds raw disk installer image (to be installed on bootable media)" + @echo " installer-iso builds an ISO installers image (to be installed on bootable media)" + @echo " installer-net builds a tarball of artifacts to be used for PXE booting" + @echo " verification-raw builds raw disk verification image (to be installed on bootable media)" + @echo " verification-net builds a tarball of artifacts to be used for PXE verification" + @echo " verification-iso builds an ISO verification image (to be installed on bootable media)" @echo @echo "Commonly used run targets (note they don't automatically rebuild images they run):" @echo " run-compose runs all EVE microservices via docker-compose deployment" From 2c605adbe6d209c7547729bd49edd729bc1cd665 Mon Sep 17 00:00:00 2001 From: Roman Penyaev Date: Mon, 15 Apr 2024 12:31:09 +0200 Subject: [PATCH 6/6] docs/BUILD.md: describe process of building live image with LIVE_PKG 1. Creating a live image with an EXT4 root filesystem: `make ROOTFS_FORMAT=ext4 live` 2. Patch the EXT4 partition of the live image with only the necessary package, e.g. pkg/pillar: `make LIVE_PKG=pkg/pillar live` Signed-off-by: Roman Penyaev --- docs/BUILD.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/BUILD.md b/docs/BUILD.md index 6281f8f3be..200e5d3641 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -377,6 +377,22 @@ The current process uses the control knob `PLATFORM` in the following places in * As the first argument to [prepare-platform.sh](../tools/prepare-platform.sh) in the Makefile targets for `live.*`, `installer.*` and `verification.*`, which, in turn, uses it to add specific files to the build and output directories prior to making the final image. * In the final eve image `lf-edge/eve`, specifically the entrypoint [runme.sh](../pkg/eve/runme.sh), where the platform is passed as `-p` argument when calling `docker run`, and is used to modify the final layout. +### Note for a developer + +`make live` typically requires a significant amount of time, approximately 3 minutes, to generate a tarball and the resulting squashfs image. This delay can be frustrating during daily development cycles of code compilation, live testing, and iteration. For developers who are less patient, optimizing the live image build process is essential. This optimization entails a few adjustments to the build procedure: + +1. Creating a live image with an EXT4 root filesystem. The live command also supports the EXT4 format for the root filesystem, resulting in a larger image size. However, this format allows for a mutable root filesystem, enabling it to be remounted with read-write (RW) options. This flexibility provides greater freedom during EVE runtime as well as during the live image creation, facilitating on-the-fly modifications to the image instead of requiring a complete rebuild from scratch. First you need to create a live image with EXT4 as the following: + + `make ROOTFS_FORMAT=ext4 live` + +1. To streamline the generation of an up-to-date live image, especially during iterative development cycles, you can patch the EXT4 partition with only the necessary package, such as `pkg/pillar`. This approach minimizes the time required for generating the live image. Below is the command for patching the EXT4 live image generated by the previous command: + + `make LIVE_PKG=pkg/pillar live` + + The command generates the rootfs tarball consisting only from the `pkg/pillar` and patches the live image using the `guestfish` one-liner (check the main Makefile for details). + +This approach reduces creation of the resulting live image from 3 minutes to 20 seconds (approximately). + ### Generating any yml Note that the process above can be used to generate any yml file, not just `rootfs-$(HV).yml`. As long as