buildextend-installer: ship live PXE artifacts inside live ISO

src/cmd-buildextend-installer

@@ -48,8 +48,6 @@ parser.add_argument("--force", action='store_true', default=False,
                     help="Overwrite previously generated installer")
 parser.add_argument("--legacy-pxe", action='store_true', default=False,
                     help="Generate stub PXE rootfs image")
-parser.add_argument("--no-pxe", action='store_true', default=False,
-                    help="Do not generate PXE media")
 args = parser.parse_args()
 
 # Identify the builds and target the latest build if none provided
@@ -108,44 +106,71 @@ if os.path.isdir(tmpdir):
 
 tmpisoroot = os.path.join(tmpdir, image_type)
 tmpisoimages = os.path.join(tmpisoroot, 'images')
+tmpisoimagespxe = os.path.join(tmpisoimages, 'pxeboot')
 tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux')
 # contents of initramfs on both PXE and ISO
-tmpinitrd_base = os.path.join(tmpdir, 'initrd-base')
-# additional contents of initramfs on ISO
-tmpinitrd_iso = os.path.join(tmpdir, 'initrd-iso')
-# additional contents of initramfs on PXE
-tmpinitrd_pxe = os.path.join(tmpdir, 'initrd-pxe')
-# contents of PXE rootfs image
-tmpinitrd_pxe_rootfs = os.path.join(tmpdir, 'initrd-pxe-rootfs')
-
-for d in (tmpdir, tmpisoroot, tmpisoimages, tmpisoisolinux, tmpinitrd_base,
-        tmpinitrd_iso, tmpinitrd_pxe, tmpinitrd_pxe_rootfs):
+tmpinitrd_base = os.path.join(tmpdir, 'initrd')
+# contents of rootfs image
+tmpinitrd_rootfs = os.path.join(tmpdir, 'initrd-rootfs')
+
+for d in (tmpdir, tmpisoroot, tmpisoimages, tmpisoimagespxe, tmpisoisolinux,
+        tmpinitrd_base, tmpinitrd_rootfs):
     os.mkdir(d)
 
 # Number of padding bytes at the end of the ISO initramfs for embedding
 # an Ignition config
 initrd_ignition_padding = 256 * 1024
 
 
+# The kernel requires that uncompressed cpio archives appended to an initrd
+# start on a 4-byte boundary.  If there's misalignment, it stops unpacking
+# and says:
+#
+#     Initramfs unpacking failed: invalid magic at start of compressed archive
+#
+# Append NUL bytes to destf until its size is a multiple of 4 bytes.
+#
+# https://www.kernel.org/doc/Documentation/early-userspace/buffer-format.txt
+# https://github.com/torvalds/linux/blob/47ec5303/init/initramfs.c#L463
+def align_initrd_for_uncompressed_append(destf):
+    offset = destf.tell()
+    if offset % 4:
+        destf.write(b'\0' * (4 - offset % 4))
+
+
 # https://www.kernel.org/doc/html/latest/admin-guide/initrd.html#compressed-cpio-images
-def mkinitrd_pipe(tmproot, destf):
-    findproc = subprocess.Popen(['find', '.', '-mindepth', '1', '-print0'],
-                                cwd=tmproot, stdout=subprocess.PIPE)
+def mkinitrd_pipe(tmproot, destf, compress=True):
+    if not compress:
+        align_initrd_for_uncompressed_append(destf)
+    files = subprocess.check_output(['find', '.', '-mindepth', '1', '-print0'],
+                                cwd=tmproot)
+    file_list = files.split(b'\0')
+    # If there's a root.squashfs, it _must_ be the first file in the cpio
+    # archive, since the dracut 20live module assumes its contents are at
+    # a fixed offset in the archive.
+    squashfs = b'./root.squashfs'
+    if squashfs in file_list:
+        file_list.remove(squashfs)
+        file_list.insert(0, squashfs)
     cpioproc = subprocess.Popen(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
             '--quiet', '--reproducible', '--force-local', '--null',
-            '-D', tmproot], stdin=findproc.stdout, stdout=subprocess.PIPE)
-    # Almost everything we're adding is already compressed.  The kernel
-    # requires us to compress again, but don't try very hard.
-    gzipargs = ['gzip', '-1']
+            '-D', tmproot], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    if compress:
+        gzipargs = ['gzip', '-9']
+    else:
+        gzipargs = ['cat']
     gzipproc = subprocess.Popen(gzipargs, stdin=cpioproc.stdout, stdout=destf)
+    cpioproc.stdin.write(b'\0'.join(file_list))
+    cpioproc.stdin.close()
     assert cpioproc.wait() == 0, f"cpio exited with {cpioproc.returncode}"
-    assert findproc.wait() == 0, f"find exited with {findproc.returncode}"
     assert gzipproc.wait() == 0, f"gzip exited with {gzipproc.returncode}"
+    # Fix up padding so the user can append the rootfs afterward
+    align_initrd_for_uncompressed_append(destf)
 
 
-def extend_initrd(initramfs, tmproot):
+def extend_initrd(initramfs, tmproot, compress=True):
     with open(initramfs, 'ab') as fdst:
-        mkinitrd_pipe(tmproot, fdst)
+        mkinitrd_pipe(tmproot, fdst, compress=compress)
 
 
 def cp_reflink(src, dest):
@@ -180,7 +205,10 @@ def file_offset_in_iso(isoinfo, filename):
 def generate_iso():
     # convention for kernel and initramfs names
     kernel_img = 'vmlinuz'
-    initramfs_img = 'initramfs.img'
+    initrd_img = 'initrd.img'
+    # other files
+    rootfs_img = 'rootfs.img'
+    ignition_img = 'ignition.img'
 
     tmpisofile = os.path.join(tmpdir, iso_name)
 
@@ -206,12 +234,18 @@ def generate_iso():
     moduledir = process.stdout.decode().split('\0')[1]
 
     # copy those files out of the ostree into the iso root dir
+    initramfs_img = 'initramfs.img'
     for file in [kernel_img, initramfs_img]:
         run_verbose(['/usr/bin/ostree', 'checkout', '--force-copy', '--repo', repo,
                      '--user-mode', '--subpath', os.path.join(moduledir, file),
-                     f"{buildmeta_commit}", tmpisoimages])
+                     f"{buildmeta_commit}", tmpisoimagespxe])
         # initramfs isn't world readable by default so let's open up perms
-        os.chmod(os.path.join(tmpisoimages, file), 0o644)
+        os.chmod(os.path.join(tmpisoimagespxe, file), 0o644)
+        if file == initramfs_img:
+            os.rename(
+                os.path.join(tmpisoimagespxe, initramfs_img),
+                os.path.join(tmpisoimagespxe, initrd_img)
+            )
 
     # Generate initramfs stamp file which says whether it's a live or legacy
     # initramfs.  Store the build ID in it.
@@ -228,19 +262,20 @@ def generate_iso():
     # rootfs has been appended and confirming that initramfs and rootfs are
     # from the same build.
     if is_live:
-        stamppath = os.path.join(tmpinitrd_pxe_rootfs, 'etc/coreos-live-rootfs')
+        stamppath = os.path.join(tmpinitrd_rootfs, 'etc/coreos-live-rootfs')
         os.makedirs(os.path.dirname(stamppath), exist_ok=True)
         with open(stamppath, 'w') as fh:
             fh.write(args.build + '\n')
 
+    # Add Ignition padding file to ISO image
+    if is_live:
+        with open(os.path.join(tmpisoimages, ignition_img), 'wb') as fdst:
+            fdst.write(bytes(initrd_ignition_padding))
+
     # Add osmet files
-    if args.legacy_pxe:
-        tmpinitrd_pxe_or_rootfs = tmpinitrd_pxe
-    else:
-        tmpinitrd_pxe_or_rootfs = tmpinitrd_pxe_rootfs
     if is_live:
-        tmp_osmet = os.path.join(tmpinitrd_iso, img_metal_obj['path'] + '.osmet')
-        tmp_osmet4k = os.path.join(tmpinitrd_iso, img_metal4k_obj['path'] + '.osmet')
+        tmp_osmet = os.path.join(tmpinitrd_rootfs, img_metal_obj['path'] + '.osmet')
+        tmp_osmet4k = os.path.join(tmpinitrd_rootfs, img_metal4k_obj['path'] + '.osmet')
         print(f'Generating osmet file for 512b metal image')
         run_verbose(['/usr/lib/coreos-assembler/osmet-pack',
                      img_metal, '512', tmp_osmet, img_metal_checksum,
@@ -249,42 +284,54 @@ def generate_iso():
         run_verbose(['/usr/lib/coreos-assembler/osmet-pack',
                      img_metal4k, '4096', tmp_osmet4k, img_metal4k_checksum,
                      'fast' if args.fast else 'normal'])
-        if not args.no_pxe:
-            cp_reflink(tmp_osmet, os.path.join(tmpinitrd_pxe_or_rootfs, os.path.basename(tmp_osmet)))
-            cp_reflink(tmp_osmet4k, os.path.join(tmpinitrd_pxe_or_rootfs, os.path.basename(tmp_osmet4k)))
 
     # Generate root squashfs
-    tmp_squashfs = None
     if is_live:
         print(f'Compressing squashfs with {squashfs_compression}')
-        tmp_squashfs = os.path.join(tmpisoroot, 'root.squashfs')
+        # Name must be exactly "root.squashfs" because the 20live dracut
+        # module makes assumptions about the length of the name in
+        # sysroot.mount
+        tmp_squashfs = os.path.join(tmpinitrd_rootfs, 'root.squashfs')
         run_verbose(['/usr/lib/coreos-assembler/gf-mksquashfs',
                     img_metal, tmp_squashfs, squashfs_compression])
-        if not args.no_pxe:
-            cp_reflink(tmp_squashfs, os.path.join(tmpinitrd_pxe_or_rootfs, 'root.squashfs'))
 
-    # Generate initramfses
+    # Generate rootfs image
+    if is_live:
+        iso_rootfs = os.path.join(tmpisoimagespxe, rootfs_img)
+        # The rootfs must be uncompressed because the ISO mounts
+        # root.squashfs directly from the middle of the file
+        extend_initrd(iso_rootfs, tmpinitrd_rootfs, compress=False)
+        # Check that the root.squashfs magic number is in the offset
+        # hardcoded in sysroot.mount in 20live/live-generator
+        with open(iso_rootfs, 'rb') as fh:
+            fh.seek(124)
+            if fh.read(4) != b'hsqs':
+                raise Exception("root.squashfs not at expected offset in rootfs image")
+        pxe_rootfs = os.path.join(tmpdir, rootfs_img)
+        # This is super-messy but it's temporary.
+        if args.legacy_pxe:
+            tmpinitrd_legacy = os.path.join(tmpdir, 'initrd-legacy')
+            os.makedirs(os.path.join(tmpinitrd_legacy, 'etc'))
+            os.rename(
+                os.path.join(tmpinitrd_rootfs, 'etc/coreos-live-rootfs'),
+                os.path.join(tmpinitrd_legacy, 'etc/coreos-live-rootfs')
+            )
+            extend_initrd(pxe_rootfs, tmpinitrd_legacy)
+        else:
+            # Clone to PXE image
+            cp_reflink(iso_rootfs, pxe_rootfs)
+        # Save stream hash of rootfs for verifying out-of-band fetches
+        os.makedirs(os.path.join(tmpinitrd_base, 'etc'), exist_ok=True)
+        make_stream_hash(pxe_rootfs, os.path.join(tmpinitrd_base, 'etc/coreos-live-want-rootfs'))
     # Add common content
-    iso_initramfs = os.path.join(tmpisoimages, initramfs_img)
+    iso_initramfs = os.path.join(tmpisoimagespxe, initrd_img)
     extend_initrd(iso_initramfs, tmpinitrd_base)
-    if not args.no_pxe:
-        # Clone to PXE image
-        pxe_initramfs = os.path.join(tmpdir, initramfs_img)
-        cp_reflink(iso_initramfs, pxe_initramfs)
-        # Generate rootfs image
-        if is_live:
-            pxe_rootfs = os.path.join(tmpdir, 'rootfs.img')
-            extend_initrd(pxe_rootfs, tmpinitrd_pxe_rootfs)
-            # Save stream hash of rootfs for verifying out-of-band fetches
-            os.makedirs(os.path.join(tmpinitrd_pxe, 'etc'), exist_ok=True)
-            make_stream_hash(pxe_rootfs, os.path.join(tmpinitrd_pxe, 'etc/coreos-live-want-rootfs'))
-        # Add additional content to PXE image
-        extend_initrd(pxe_initramfs, tmpinitrd_pxe)
-    # Add additional content to ISO image
-    extend_initrd(iso_initramfs, tmpinitrd_iso)
-    # Add Ignition padding to ISO image
-    with open(iso_initramfs, 'ab') as fdst:
-        fdst.write(bytes(initrd_ignition_padding))
+    # Clone to PXE image
+    pxe_initramfs = os.path.join(tmpdir, initrd_img)
+    cp_reflink(iso_initramfs, pxe_initramfs)
+    # Put the rootfs contents in the initramfs in the legacy PXE case
+    if is_live and args.legacy_pxe:
+        extend_initrd(pxe_initramfs, tmpinitrd_rootfs)
 
     # Read and filter kernel arguments for substituting into ISO bootloader
     result = run_verbose(['/usr/lib/coreos-assembler/gf-get-kargs',
@@ -378,12 +425,9 @@ def generate_iso():
 
         # s390x's z/VM CMS files are limited to 8 char for filenames and extensions
         # Also it is nice to keep naming convetion with Fedora/RHEL for existing users and code
-        kernel_dest = os.path.join(tmpisoimages, 'kernel.img')
-        shutil.move(os.path.join(tmpisoimages, kernel_img), kernel_dest)
+        kernel_dest = os.path.join(tmpisoimagespxe, 'kernel.img')
+        shutil.move(os.path.join(tmpisoimagespxe, kernel_img), kernel_dest)
         kernel_img = 'kernel.img'
-        iso_initramfs_dest = os.path.join(tmpisoimages, 'initrd.img')
-        shutil.move(iso_initramfs, iso_initramfs_dest)
-        iso_initramfs = iso_initramfs_dest
 
         # combine kernel, initramfs and cmdline using lorax/mk-s390-cdboot tool
         run_verbose(['/usr/bin/mk-s390-cdboot',
@@ -459,12 +503,13 @@ def generate_iso():
     isoinfo = run_verbose(['isoinfo', '-lR', '-i', tmpisofile],
                           stdout=subprocess.PIPE, text=True).stdout
 
-    # We've already padded the ISO initrd with initrd_ignition_padding bytes of
-    # zeroes.  Find the byte offset of that padding within the ISO image and
-    # write it into a custom header at the end of the ISO 9660 System Area,
-    # which is 32 KB at the start of the image "reserved for system use".
-    # The System Area usually contains partition tables and the like, and
-    # we're assuming that none of our platforms use the last 24 bytes of it.
+    # We've already created a file in the ISO with initrd_ignition_padding
+    # bytes of zeroes.  Find the byte offset of that file within the ISO
+    # image and write it into a custom header at the end of the ISO 9660
+    # System Area, which is 32 KB at the start of the image "reserved for
+    # system use".  The System Area usually contains partition tables and
+    # the like, and we're assuming that none of our platforms use the last
+    # 24 bytes of it.
     #
     # This allows an external tool, `coreos-installer iso embed`, to modify
     # an existing ISO image to embed a user's custom Ignition config.
@@ -476,12 +521,8 @@ def generate_iso():
     # Skip on s390x because that platform uses an embedded El Torito image
     # with its own copy of the initramfs.
     if is_live and basearch != "s390x":
-        # Start of the initramfs within the ISO
-        offset = file_offset_in_iso(isoinfo, initramfs_img)
-        # End of the initramfs within the ISO
-        offset += os.stat(iso_initramfs).st_size
-        # Start of the initramfs padding
-        offset -= initrd_ignition_padding
+        # Start of the Ignition padding within the ISO
+        offset = file_offset_in_iso(isoinfo, ignition_img)
         with open(tmpisofile, 'r+b') as isofh:
             # Verify that the calculated byte range is empty
             isofh.seek(offset)
@@ -502,33 +543,32 @@ def generate_iso():
     })
     shutil.move(tmpisofile, f"{builddir}/{iso_name}")
 
-    if not args.no_pxe:
-        kernel_name = f'{base_name}-{args.build}-{image_type}-kernel-{basearch}'
-        initramfs_name = f'{base_name}-{args.build}-{image_type}-initramfs.{basearch}.img'
-        kernel_file = os.path.join(builddir, kernel_name)
-        initramfs_file = os.path.join(builddir, initramfs_name)
-        shutil.copyfile(os.path.join(tmpisoimages, kernel_img), kernel_file)
-        shutil.move(pxe_initramfs, initramfs_file)
+    kernel_name = f'{base_name}-{args.build}-{image_type}-kernel-{basearch}'
+    initramfs_name = f'{base_name}-{args.build}-{image_type}-initramfs.{basearch}.img'
+    kernel_file = os.path.join(builddir, kernel_name)
+    initramfs_file = os.path.join(builddir, initramfs_name)
+    shutil.copyfile(os.path.join(tmpisoimagespxe, kernel_img), kernel_file)
+    shutil.move(pxe_initramfs, initramfs_file)
+    buildmeta['images'].update({
+        meta_keys['kernel']: {
+            'path': kernel_name,
+            'sha256': sha256sum_file(kernel_file)
+        },
+        meta_keys['initramfs']: {
+            'path': initramfs_name,
+            'sha256': sha256sum_file(initramfs_file)
+        }
+    })
+    if is_live:
+        rootfs_name = f'{base_name}-{args.build}-{image_type}-rootfs.{basearch}.img'
+        rootfs_file = os.path.join(builddir, rootfs_name)
+        shutil.move(pxe_rootfs, rootfs_file)
         buildmeta['images'].update({
-            meta_keys['kernel']: {
-                'path': kernel_name,
-                'sha256': sha256sum_file(kernel_file)
-            },
-            meta_keys['initramfs']: {
-                'path': initramfs_name,
-                'sha256': sha256sum_file(initramfs_file)
+            'live-rootfs': {
+                'path': rootfs_name,
+                'sha256': sha256sum_file(rootfs_file)
             }
         })
-        if is_live:
-            rootfs_name = f'{base_name}-{args.build}-{image_type}-rootfs.{basearch}.img'
-            rootfs_file = os.path.join(builddir, rootfs_name)
-            shutil.move(pxe_rootfs, rootfs_file)
-            buildmeta['images'].update({
-                'live-rootfs': {
-                    'path': rootfs_name,
-                    'sha256': sha256sum_file(rootfs_file)
-                }
-            })
 
     write_json(buildmeta_path, buildmeta)
     print(f"Updated: {buildmeta_path}")