Add support for reading and writing lockfiles

First, add `manifest-lock.json` to the list of magical files in the
source config which we check for.

Then, teach `fetch` and `build` to respect the lockfile by default, but
add an `--update-lockfile` option to the former allow bumping the
lockfile.

Finally, always generate an output lockfile from the compose and put in
the build dir as `manifest-lock.generated.json`.

This is the first step towards actually being able to make use of
lockfiles in the FCOS pipeline.

Author: jlebon

src/cmd-build

@@ -151,11 +151,23 @@ EOF
 
 prepare_git_artifacts "${configdir_gitrepo}" "${PWD}/coreos-assembler-config.tar.gz" "${PWD}/coreos-assembler-config-git.json"
 
+lock_arg=
+if [ -f "${manifest_lock}" ]; then
+    lock_arg="--ex-lockfile=${manifest_lock}"
+    echo "Building from lockfile ${manifest_lock}"
+    sleep 1
+fi
+
 # These need to be absolute paths right now for rpm-ostree
 composejson=${PWD}/tmp/compose.json
+# Put this under tmprepo so it gets automatically chown'ed if needed
+lockfile_out=${tmprepo}/tmp/manifest-lock.generated.json
 # --cache-only is here since `fetch` is a separate verb.
+# shellcheck disable=SC2086
 runcompose --cache-only ${FORCE} --add-metadata-from-json "${commitmeta_input_json}" \
-           --write-composejson-to "${composejson}"
+           --write-composejson-to "${composejson}" \
+           --ex-write-lockfile-to "${lockfile_out}" \
+           ${lock_arg}
 # Very special handling for --write-composejson-to as rpm-ostree doesn't
 # write it if the commit didn't change.
 if [ -f "${changed_stamp}" ]; then
@@ -306,6 +318,9 @@ if [ -n "${ref_is_temp}" ]; then
     mv meta.json{.new,}
 fi
 
+# Move lockfile into build dir
+mv "${lockfile_out}" .
+
 # And add the commit metadata itself, which includes notably the rpmdb pkglist
 # in a format that'd be easy to generate diffs out of for higher level tools
 "${dn}"/commitmeta_to_json "${tmprepo}" "${commit}" > commitmeta.json

src/cmd-fetch

@@ -8,14 +8,15 @@ dn=$(dirname "$0")
 print_help() {
     cat 1>&2 <<'EOF'
 Usage: coreos-assembler fetch --help
-       coreos-assembler fetch
+       coreos-assembler fetch [--update-lockfile]
 
   Fetch and import the latest packages.
 EOF
 }
 
+UPDATE_LOCKFILE=
 rc=0
-options=$(getopt --options h --longoptions help -- "$@") || rc=$?
+options=$(getopt --options h --longoptions help,update-lockfile -- "$@") || rc=$?
 [ $rc -eq 0 ] || {
     print_help
     exit 1
@@ -27,6 +28,9 @@ while true; do
             print_help
             exit 0
             ;;
+        --update-lockfile)
+            UPDATE_LOCKFILE=1
+            ;;
         --)
             shift
             break
@@ -46,4 +50,21 @@ if [ $# -ne 0 ]; then
 fi
 
 prepare_build
-runcompose --download-only
+
+lock_arg=
+if [ -n "${UPDATE_LOCKFILE}" ]; then
+    # Put this under tmprepo so it gets automatically chown'ed if needed
+    lock_arg="--ex-write-lockfile-to=${tmprepo}/tmp/manifest-lock.json"
+elif [ -f "${manifest_lock}" ]; then
+    lock_arg="--ex-lockfile=${manifest_lock}"
+    echo "Fetching RPMs from lockfile ${manifest_lock}"
+    sleep 1
+fi
+
+# shellcheck disable=SC2086
+runcompose --download-only ${lock_arg}
+
+if [ -n "${UPDATE_LOCKFILE}" ]; then
+    mv "${tmprepo}/tmp/manifest-lock.json" "${manifest_lock}"
+    echo "Updated lockfile ${manifest_lock}"
+fi

src/cmdlib.sh

@@ -166,7 +166,8 @@ prepare_build() {
     workdir="$(pwd)"
     configdir=${workdir}/src/config
     manifest=${configdir}/manifest.yaml
-    export workdir configdir manifest
+    manifest_lock=${configdir}/manifest-lock.json
+    export workdir configdir manifest manifest_lock
 
     if ! [ -f "${manifest}" ]; then
         fatal "Failed to find ${manifest}"