From 036197f6e12192b850f8a7f0266663fd959140ae Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 16 May 2019 10:07:28 -0400 Subject: [PATCH] Move OSTree commit into build directory Rather than keeping OSTree data separately in the toplevel `repo/`, make it part of the build directory. This solves a bunch of issues and makes things conceptually clearer. See discussions in: https://github.com/coreos/coreos-assembler/issues/159 --- src/cmd-build | 75 ++++++++++++++++++++++++++++------- src/cmd-buildextend-installer | 16 +++++--- src/cmd-buildextend-metal | 20 +++++++++- src/cmd-clean | 3 +- src/cmd-init | 1 - src/cmdlib.py | 20 ++++++++++ src/cmdlib.sh | 23 ++++++++--- src/prune_builds | 17 -------- 8 files changed, 128 insertions(+), 47 deletions(-) diff --git a/src/cmd-build b/src/cmd-build index 9ff770bf9a..ba6046e547 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -79,15 +79,37 @@ if [ -L "${workdir}"/builds/latest ]; then fi previous_commit= -if [ -n "${ref:-}" ]; then - previous_commit=$(ostree --repo="${workdir}"/repo rev-parse "${ref}" 2>/dev/null || true) -fi -# If the ref was unset or missing, look at the previous build -if [ -z "${previous_commit}" ] && [ -n "${previous_build}" ]; then +if [ -n "${previous_build}" ]; then previous_commit=$(jq -r '.["ostree-commit"]' < "${previous_builddir}/meta.json") fi echo "Previous commit: ${previous_commit:-none}" +if [ -n "${previous_commit}" ]; then + # If we don't have the previous commit (or it's partial), then try to + # re-import it; this saves us recompression time later on since it's likely + # a lot of the new objects in this build will be the same. + commitpath=${tmprepo}/objects/${previous_commit::2}/${previous_commit:2}.commit + commitpartial=${tmprepo}/state/${previous_commit}.commitpartial + if [ ! -f "${commitpath}" ] || [ -f "${commitpartial}" ]; then + if [ -f "${previous_builddir}/ostree-commit.tar" ]; then + mkdir tmp/prev_repo + tar -C tmp/prev_repo -xf "${previous_builddir}/ostree-commit.tar" + ostree pull-local --repo="${tmprepo}" tmp/prev_repo "${previous_commit}" + rm -rf tmp/prev_repo + else + # ok, just fallback to importing the commit object only + mkdir -p "$(dirname "${commitpath}")" + cp "${previous_builddir}/ostree-commit-object" "${commitpath}" + touch "${commitpartial}" + fi + fi + + # and point the ref to it if there isn't one already (in which case it might be newer, but e.g. Anaconda failed) + if ! ostree rev-parse --repo="${tmprepo}" "${ref}" &>/dev/null; then + ostree refs --repo="${tmprepo}" --create "${ref}" "${previous_commit}" + fi +fi + # Calculate image input checksum now and gather previous image build variables if any ks_path="${configdir}"/image.ks if [ -f "${ks_path}" ]; then @@ -125,8 +147,6 @@ composejson=${PWD}/tmp/compose.json # --cache-only is here since `fetch` is a separate verb. runcompose --cache-only ${FORCE} --add-metadata-from-json "${commitmeta_input_json}" \ --write-composejson-to "${composejson}" -# Always update the summary, since we used to do so -ostree --repo="${workdir}/repo" summary -u # Very special handling for --write-composejson-to as rpm-ostree doesn't # write it if the commit didn't change. if [ -f "${changed_stamp}" ]; then @@ -136,7 +156,10 @@ if [ -f "${changed_stamp}" ]; then # Save this in case the image build fails cp -a --reflink=auto "${composejson}" "${workdir}"/tmp/compose-"${commit}".json else - commit=${previous_commit} + # Pick up from tmprepo; that's what rpm-ostree is comparing against. It may + # be the same as previous_commit, or newer if a previous build failed image + # creation. + commit=$(ostree rev-parse --repo="${tmprepo}" "${ref}") image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str) # Note we may not actually have a previous build in the case of # successfully composing an ostree but failing the image on the @@ -148,7 +171,7 @@ else # Grab the previous treecompose JSON (local developer case: treecompose succeeded but # image build failed) if possible, otherwise grab the previous build - cached_previous_composejson="${workdir}"/tmp/compose-"${previous_commit}".json + cached_previous_composejson=${workdir}/tmp/compose-${commit}.json if [ -f "${cached_previous_composejson}" ]; then echo "Resuming partial build from: ${commit}" cp -a --reflink=auto "${cached_previous_composejson}" "${composejson}" @@ -167,12 +190,17 @@ else fi if [ -n "${previous_build}" ]; then - rpm-ostree --repo="${workdir}"/repo db diff "${previous_commit}" "${commit}" + # do it once for the terminal + rpm-ostree --repo="${tmprepo}" db diff "${previous_commit}" "${commit}" + # and once more for the metadata, but only keep the pkgdiff key + rpm-ostree --repo="${tmprepo}" db diff --format=json \ + "${previous_commit}" "${commit}" | \ + jq '{pkgdiff: .pkgdiff}' > tmp/diff.json fi image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str) echo "New image input checksum: ${image_input_checksum}" -version=$(ostree --repo="${workdir}"/repo show --print-metadata-key=version "${commit}" | sed -e "s,',,g") +version=$(ostree --repo="${tmprepo}" show --print-metadata-key=version "${commit}" | sed -e "s,',,g") if [ "${previous_commit}" = "${commit}" ] && [ -n "${previous_image_genver:-}" ]; then image_genver=$((previous_image_genver + 1)) buildid="${version}"-"${image_genver}" @@ -192,10 +220,10 @@ img_base=tmp/${imageprefix}-base.qcow2 checksum_location=$(find /usr/lib/coreos-assembler-anaconda/ -name '*CHECKSUM' | head -1) img_qemu=${imageprefix}-qemu.qcow2 -run_virtinstall "${workdir}/repo" "${ref}" "${PWD}"/"${img_base}" --variant=cloud +run_virtinstall "${tmprepo}" "${ref}" "${PWD}"/"${img_base}" --variant=cloud /usr/lib/coreos-assembler/gf-platformid "$(pwd)"/"${img_base}" "$(pwd)"/"${img_qemu}" qemu -"${dn}"/write-commit-object "${workdir}/repo" "${commit}" "$(pwd)" +"${dn}"/write-commit-object "${tmprepo}" "${commit}" "$(pwd)" build_timestamp=$(date -u +$RFC3339) vm_iso_checksum=$(awk '/SHA256.*iso/{print$NF}' "${checksum_location}") @@ -253,7 +281,7 @@ fi # Merge all the JSON; note that we want ${composejson} first # since we may be overriding data from a previous build. -cat "${composejson}" tmp/meta.json tmp/images.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json +cat "${composejson}" tmp/meta.json tmp/diff.json tmp/images.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json # Filter out `ref` if it's temporary if [ -n "${ref_is_temp}" ]; then @@ -263,12 +291,29 @@ fi # And add the commit metadata itself, which includes notably the rpmdb pkglist # in a format that'd be easy to generate diffs out of for higher level tools -"${dn}"/commitmeta_to_json "${workdir}/repo" "${commit}" > commitmeta.json +"${dn}"/commitmeta_to_json "${tmprepo}" "${commit}" > commitmeta.json + +# And create the ostree repo tarball containing the commit +if [ "${commit}" == "${previous_commit}" ]; then + cp -a --reflink=auto "${previous_builddir}/ostree-commit.tar" . +else + ostree init --repo=repo --mode=archive + ostree pull-local --repo=repo "${tmprepo}" "${ref}" + if [ -n "${ref_is_temp}" ]; then + ostree refs --repo=repo --delete "${ref}" + fi + # Don't compress; archive repos are already compressed individually and we'd + # gain ~20M at best. We could probably have better gains if we compress the + # whole repo in bare/bare-user mode, but that's a different story... + tar -cf ostree-commit.tar -C repo . + rm -rf repo +fi # Clean up our temporary data saved_build_tmpdir="${workdir}/tmp/last-build-tmp" rm -rf "${saved_build_tmpdir}" mv -T tmp "${saved_build_tmpdir}" +ostree prune --repo="${tmprepo}" --refs-only # Back to the toplevel build directory, so we can rename this one cd "${workdir}"/builds # We create a .build-commit file to note that we're in the diff --git a/src/cmd-buildextend-installer b/src/cmd-buildextend-installer index eebab196d0..fcb43a1ad3 100755 --- a/src/cmd-buildextend-installer +++ b/src/cmd-buildextend-installer @@ -14,6 +14,7 @@ import tempfile sys.path.insert(0, '/usr/lib/coreos-assembler') from cmdlib import run_verbose, write_json, sha256sum_file +from cmdlib import import_ostree_commit # Parse args and dispatch parser = argparse.ArgumentParser() @@ -36,6 +37,11 @@ buildmeta_path = os.path.join(builddir, 'meta.json') with open(buildmeta_path) as f: buildmeta = json.load(f) +# Grab the commit hash for this build +buildmeta_commit = buildmeta['ostree-commit'] + +repo = os.path.join(workdir, 'tmp/repo') + # Don't run if it's already been done, unless forced if 'iso' in buildmeta['images'] and not args.force: print(f"Installer has already been built for {args.build}. Skipping.") @@ -68,20 +74,17 @@ def generate_iso(): tmpisofile = os.path.join(tmpdir, iso_name) - # Grab the commit hash for this build - buildmeta_commit = buildmeta['ostree-commit'] - # Find the directory under `/usr/lib/modules/` where the # kernel/initrd live. It will be the 2nd entity output by # `ostree ls /usr/lib/modules` - process = run_verbose(['/usr/bin/ostree', '--repo=./repo', 'ls', + process = run_verbose(['/usr/bin/ostree', 'ls', '--repo', repo, '--nul-filenames-only', f"{buildmeta_commit}", '/usr/lib/modules'], capture_output=True) moduledir = process.stdout.decode().split('\0')[1] # copy those files out of the ostree into the iso root dir for file in ['initramfs.img', 'vmlinuz']: - run_verbose(['/usr/bin/ostree', '--repo=./repo', 'checkout', + run_verbose(['/usr/bin/ostree', 'checkout', '--repo', repo, '--user-mode', '--subpath', os.path.join(moduledir, file), f"{buildmeta_commit}", tmpisoimages]) # initramfs isn't world readable by default so let's open up perms @@ -199,5 +202,8 @@ def generate_iso(): print(f"Updated: {buildmeta_path}") +commit_tar = os.path.join(builddir, 'ostree-commit.tar') +import_ostree_commit(repo, buildmeta_commit, commit_tar) + # Do it! generate_iso() diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 4ab10fce0a..ef15347080 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -98,6 +98,24 @@ if [ "${ref}" = "null" ]; then ref="tmpref-${name}" ref_is_temp=1 fi +commit=$(json_key ostree-commit) + +ostree_repo=${tmprepo} +commitpath=${tmprepo}/objects/${commit::2}/${commit:2}.commit +commitpartial=${tmprepo}/state/${commit}.commitpartial +if [ ! -f "${commitpath}" ] || [ -f "${commitpartial}" ]; then + # Probably an older commit or tmp/ was wiped. Let's extract it to a separate + # temporary repo (not to be confused with ${tmprepo}...) so we can feed it + # as a ref (if not temp) to Anaconda. + mkdir -p tmp/repo + tar -C tmp/repo -xf "${builddir}/ostree-commit.tar" + ostree_repo=$PWD/tmp/repo + if [ -n "${ref_is_temp}" ]; then + # this gets promptly "dereferenced" back in run_virtinstall, but it + # keeps the code below simple so it can work in both temp/not temp cases + ostree refs --repo="${ostree_repo}" "${commit}" --create "${ref}" + fi # otherwise, the repo already has a ref, so no need to create +fi # for anaconda logs mkdir -p tmp/anaconda @@ -110,7 +128,7 @@ for itype in ${BIOS:+metal-bios} ${UEFI:+metal-uefi}; do fi path=${PWD}/${img} - run_virtinstall "${workdir}/repo" "${ref}" "${path}.tmp" --variant="${itype}" + run_virtinstall "${ostree_repo}" "${ref}" "${path}.tmp" --variant="${itype}" /usr/lib/coreos-assembler/gf-platformid "${path}"{.tmp,} metal rm -f "${path}".tmp diff --git a/src/cmd-clean b/src/cmd-clean index ea74f9a280..82b88ebf71 100755 --- a/src/cmd-clean +++ b/src/cmd-clean @@ -53,5 +53,4 @@ prepare_build cd "${workdir:?}" # Note we don't prune the cache.qcow2 or the objects # in the repo. If you want that, just rm -rf them. -rm -rf repo/refs/heads/* builds/* tmp/* -ostree --repo=repo summary -u +rm -rf builds/* tmp/* diff --git a/src/cmd-init b/src/cmd-init index f25a8ac34c..78cc396aa3 100755 --- a/src/cmd-init +++ b/src/cmd-init @@ -116,4 +116,3 @@ mkdir -p cache mkdir -p builds mkdir -p tmp mkdir -p overrides/rpm -ostree --repo=repo init --mode=archive diff --git a/src/cmdlib.py b/src/cmdlib.py index b1f95844e1..50d38b6677 100755 --- a/src/cmdlib.py +++ b/src/cmdlib.py @@ -98,6 +98,7 @@ def fatal(msg): print('fatal: {}'.format(msg), file=sys.stderr) raise SystemExit(1) + def info(msg): """ Prints info messages. @@ -137,3 +138,22 @@ def rm_allow_noent(path): os.unlink(path) except FileNotFoundError: pass + + +def import_ostree_commit(repo, commit, tarfile): + # create repo in case e.g. tmp/ was cleared out; idempotent + subprocess.check_call(['ostree', 'init', '--repo', repo, '--mode=archive']) + + # in the common case where we're operating on a recent build, the OSTree + # commit should already be in the tmprepo + commitpartial = os.path.join(repo, f'state/{commit}.commitpartial') + if (subprocess.call(['ostree', 'show', '--repo', repo, commit], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0 + and not os.path.isfile(commitpartial)): + return + + with tempfile.TemporaryDirectory(dir=f'{repo}/tmp') as d: + subprocess.check_call(['tar', '-C', d, '-xf', tarfile]) + subprocess.check_call(['ostree', 'pull-local', '--repo', repo, + d, commit]) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 458426f3fc..d38e2d2970 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -115,8 +115,8 @@ preflight() { prepare_build() { preflight - if ! [ -d repo ]; then - fatal "No $(pwd)/repo found; did you run coreos-assembler init?" + if ! [ -d builds ]; then + fatal "No $(pwd)/builds found; did you run coreos-assembler init?" elif ! has_privileges; then if [ ! -f cache/cache.qcow2 ]; then qemu-img create -f qcow2 cache/cache.qcow2 10G @@ -135,6 +135,17 @@ prepare_build() { echo "Using manifest: ${manifest}" + tmprepo=${workdir}/tmp/repo + if [ ! -d "${tmprepo}" ]; then + # backcompat: just move the toplevel repo/ + if [ -d "${workdir}/repo" ]; then + mv -T "${workdir}/repo" "${tmprepo}" + rm -f "${tmprepo}/summary" + else + ostree init --repo="${tmprepo}" --mode=archive + fi + fi + configdir_gitrepo=${configdir} if [ -e "${workdir}/src/config-git" ]; then configdir_gitrepo="${workdir}/src/config-git" @@ -142,7 +153,7 @@ prepare_build() { export configdir_gitrepo manifest_tmp_json=${workdir}/tmp/manifest.json - rpm-ostree compose tree --repo=repo --print-only "${manifest}" > "${manifest_tmp_json}" + rpm-ostree compose tree --repo="${tmprepo}" --print-only "${manifest}" > "${manifest_tmp_json}" # Abuse the rojig/name as the name of the VM images # Also grab rojig summary for image upload descriptions @@ -230,16 +241,16 @@ EOF rm -f "${changed_stamp}" # shellcheck disable=SC2086 - set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose tree --repo="${workdir}"/repo \ + set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose tree --repo="${tmprepo}" \ --cachedir="${workdir}"/cache --touch-if-changed "${changed_stamp}" \ - --unified-core "${manifest}" ${COSA_RPMOSTREE_ARGS:-} "$@" + --unified-core --no-parent "${manifest}" ${COSA_RPMOSTREE_ARGS:-} "$@" echo "Running: $*" # this is the heart of the privs vs no privs dual path if has_privileges; then sudo -E "$@" - sudo chown -R -h "${USER}":"${USER}" "${workdir}"/repo + sudo chown -R -h "${USER}":"${USER}" "${tmprepo}" else runvm "$@" fi diff --git a/src/prune_builds b/src/prune_builds index 4b4031f695..6c2b1c203c 100755 --- a/src/prune_builds +++ b/src/prune_builds @@ -174,20 +174,3 @@ if skip_pruning: for build in builds_to_delete: print(f"Pruning build {build.id}") shutil.rmtree(os.path.join(builds_dir, build.id)) - -# and finally prune OSTree repos -print(f"Pruning repo") -repo = os.path.join(args.workdir, 'repo') - -# For now, we just use the --keep-younger-than CLI here. Doing this more -# accurately would require enhancing the `prune` CLI (or just use the API -# directly?). Or we could also manually figure out the depth of the oldest -# build and then use that as the arg to `--depth`. -oldest_ostree_t = new_builds[-1].ostree_timestamp -# In a quick test, it seems like --keep-younger-than=x actually keeps commits -# with timestamps exactly equal to x, but the name is a bit tricky so let's -# be safe and just pick a time 1h before that so we're doubly sure. It might -# keep a few other older commits, but meh... -younger_than = rfc3339_time(oldest_ostree_t - timedelta(hours=1)) -subprocess.run(["ostree", "prune", "--repo", repo, "--refs-only", - f"--keep-younger-than={younger_than}"], check=True)