cmd-build: Split out qemu building into separate command

For FCOS signing, we need a way to build just the OSTree, then have it
signed, and then build the image artifacts with the signatures embedded.
To do this, we need to decouple qemu image building from `cosa build`.

This patch does this by making `cmd-buildextend-qemu` a symlink to
`cmd-buildextend-metal` and enhancing the latter to handle both cases.
A big advantage of this is that base image building is now located in a
single location.

We then extend `cosa build` to just take any arbitrary `TARGET`
arguments which become semantically equivalent to calling `cosa
buildextend-$TARGET`, where `TARGET` is one of `ostree`, `qemu`,
`metal`. (And of course, we default to building the qemu image to not
break compatibility).

Author: jlebon

Makefile

@@ -50,6 +50,7 @@ mantle:
 install:
 	install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler
 	install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler $$(find src/ -maxdepth 1 -type f)
+	cp -df -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler $$(find src/ -maxdepth 1 -type l)
 	install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib
 	install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib $$(find src/cosalib/ -maxdepth 1 -type f)
 	install -d $(DESTDIR)$(PREFIX)/bin

src/cmd-build

@@ -8,18 +8,17 @@ dn=$(dirname "$0")
 print_help() {
     cat 1>&2 <<'EOF'
 Usage: coreos-assembler build --help
-       coreos-assembler build [--force] [--skip-prune] [--version VERSION] [TARGETS]
+       coreos-assembler build [--force] [--skip-prune] [--version VERSION] [TARGET...]
 
-  Build OSTree and image artifacts from previously fetched packages.
-  The TARGETS argument is a list of artifact types to build; if unspecified it
-  defaults to "qemu".
+  Build OSTree and image base artifacts from previously fetched packages.
+  Accepted TARGET arguments:
 
-  Valid targets:
+  - ostree
+  - qemu
+  - metal
 
-    - ostree
-    - qemu
-
-  Note that all image targets also require the "ostree" target.
+  The "qemu" and "metal" targets imply "ostree". If unspecified, defaults to
+  "qemu". They are equivalent to manually running buildextend-[TARGET] after.
 EOF
 }
 
@@ -74,16 +73,24 @@ if [ $# -eq 0 ]; then
     set -- qemu
 fi
 
-build_qemu=
+# sanity check the targets and aggregate into a set
+declare -A targets=( )
 for target in "$@"; do
-    case $target in
-        ostree) ;;
-        qemu) build_qemu=1;;
-        *) fatal "Unrecognized target: $target";;
-    esac
+    if [[ $target != ostree ]]; then
+        case "$target" in
+            metal|qemu) ;;
+            *) fatal "Unrecognized target: $target" ;;
+        esac
+        targets[$target]=1
+    fi
 done
 
-export LIBGUESTFS_BACKEND=direct
+build_followup_targets() {
+    cd "${workdir}"
+    for target in "${!targets[@]}"; do
+        "/usr/lib/coreos-assembler/cmd-buildextend-${target}"
+    done
+}
 
 prepare_build
 
@@ -93,8 +100,8 @@ rpm-ostree --version
 previous_build=$(get_latest_build)
 if [ -n "${previous_build}" ]; then
     previous_builddir=$(get_build_dir "${previous_build}")
-    echo "Previous build: ${previous_build}"
 fi
+echo "Previous build: ${previous_build:-none}"
 
 previous_commit=
 if [ -n "${previous_build}" ]; then
@@ -218,6 +225,11 @@ else
     # first build.
     if [ -n "${previous_build}" ] && [ "${image_input_checksum}" = "${previous_image_input_checksum}" ]; then
         echo "No changes in image inputs."
+        # But still run through the follow-up targets. This allows us to have
+        # e.g. `cosa build metal` be idempotent even if the initial build failed
+        # for whatever reason. `buildextend-[metal|qemu]` should already be
+        # idempotent.
+        build_followup_targets
         exit 0
     fi
 
@@ -264,50 +276,6 @@ else
 fi
 echo "New build ID: ${buildid}"
 
-imageprefix="${name}"-"${buildid}"
-# Make these two verbose
-set -x
-
-build_image() {
-	local size kargs ostree_remote
-	size="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin)["size"])' < "$configdir/image.yaml")G"
-	kargs="$(python3 -c 'import sys, yaml; args = yaml.safe_load(sys.stdin).get("extra-kargs", []); print(" ".join(args))' < "$configdir/image.yaml")"
-	# use NONE here instead of empty string because this has to survive through various string processing
-	ostree_remote="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin).get("ostree-remote", "NONE"))' < "$configdir/image.yaml")"
-	save_var_subdirs="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin).get("save-var-subdirs-for-selabel-workaround", "NONE"))' < "$configdir/image.yaml")"
-	tty="console=tty0 console=${VM_TERMINAL},115200n8"
-	# tty0 does not exist on s390x
-	[ "$arch" = "s390x" ] && tty="console=${VM_TERMINAL}"
-	kargs="$kargs $tty ignition.platform.id=qemu"
-
-	qemu-img create -f qcow2 "$1" "$size"
-	runvm_with_disk "$1" qcow2 /usr/lib/coreos-assembler/create_disk.sh /dev/vda "$tmprepo" "${ref-:${commit}}" "${ostree_remote}" /usr/lib/coreos-assembler/grub.cfg "$name" "${save_var_subdirs}" "\"$kargs\""
-}
-
-echo '{}' > tmp/vm-iso-checksum.json
-if [ -n "${build_qemu}" ]; then
-    img_qemu=${imageprefix}-qemu.qcow2
-    case "$arch" in
-        "x86_64"|"aarch64"|"s390x")
-            build_image "$img_qemu"
-            ;;
-        *)
-            mkdir -p tmp/anaconda
-            # forgive me for this sin
-            checksum_location=$(find /usr/lib/coreos-assembler-anaconda/ -name '*CHECKSUM' | head -1)
-            img_base=tmp/${imageprefix}-base.qcow2
-            run_virtinstall "${tmprepo}" "${ref}" "${PWD}"/"${img_base}" --variant=cloud
-            /usr/lib/coreos-assembler/gf-platformid "$(pwd)"/"${img_base}" "$(pwd)"/"${img_qemu}" qemu
-            vm_iso_checksum=$(awk '/SHA256.*iso/{print$NF}' "${checksum_location}")
-            cat > tmp/vm-iso-checksum.json <<EOF
-{
-    "coreos-assembler.vm-iso-checksum": "${vm_iso_checksum}"
-}
-EOF
-            ;;
-    esac
-fi
-
 "${dn}"/write-commit-object "${tmprepo}" "${commit}" "$(pwd)"
 
 build_timestamp=$(date -u +$RFC3339)
@@ -340,21 +308,7 @@ cat > tmp/meta.json <<EOF
 }
 EOF
 
-if [ -n "${build_qemu}" ]; then
-    cat > tmp/images.json <<EOF
-{
-    "images": {
-        "qemu": {
-            "path": "${img_qemu}",
-            "sha256": "$(sha256sum_str < "${img_qemu}")",
-            "size": $(stat -c '%s' "${img_qemu}")
-        }
-    }
-}
-EOF
-else
-    echo '{ "images": {} }' > tmp/images.json
-fi
+echo '{ "images": {} }' > tmp/images.json
 
 # And the build information about our container, if we are executing
 # from a container.
@@ -368,7 +322,7 @@ fi
 
 # Merge all the JSON; note that we want ${composejson} first
 # since we may be overriding data from a previous build.
-cat "${composejson}" tmp/meta.json tmp/diff.json tmp/images.json tmp/vm-iso-checksum.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json
+cat "${composejson}" tmp/meta.json tmp/diff.json tmp/images.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json
 
 # Filter out `ref` if it's temporary
 if [ -n "${ref_is_temp}" ]; then
@@ -429,3 +383,6 @@ else
     "${dn}"/prune_builds --workdir "${workdir}"
 fi
 rm builds/.build-commit
+
+# and finally, build the specified targets
+build_followup_targets

src/cmd-buildextend-metal

@@ -5,10 +5,18 @@ dn=$(dirname "$0")
 # shellcheck source=src/cmdlib.sh
 . "${dn}"/cmdlib.sh
 
+# This script is used for creating both the bare metal and the canonical VM
+# image (qemu). `buildextend-qemu` is a symlink to `buildextend-metal`.
+case "$(basename "$0")" in
+    "cmd-buildextend-metal") image_type=metal;;
+    "cmd-buildextend-qemu") image_type=qemu;;
+    *) fatal "called as unexpected name $0";;
+esac
+
 print_help() {
-    cat 1>&2 <<'EOF'
-Usage: coreos-assembler buildextend-metal --help
-       coreos-assembler buildextend-metal [--build ID]
+    cat 1>&2 <<EOF
+Usage: coreos-assembler buildextend-${image_type} --help
+       coreos-assembler buildextend-${image_type} [--build ID]
 
   Build a bare metal image.
 EOF
@@ -54,12 +62,16 @@ if [ $# -ne 0 ]; then
 fi
 
 case "$arch" in
-    "x86_64"|"aarch64"|"s390x")
-        ## fall through to the rest of the file
-	;;
+    "x86_64"|"aarch64"|"s390x") use_anaconda=;;
     *)
-        echo "$arch is not supported for this command"
-        exit 1
+        # for qemu, we can fallback to Anaconda
+        if [[ ${image_type} == qemu ]]; then
+            use_anaconda=1
+        else
+            # otherwise, we don't know how to create bare metal images for this
+            # architecture
+            fatal "$arch is not supported for this command"
+        fi
         ;;
 esac
 
@@ -112,49 +124,75 @@ if [ "${rev_parsed}" != "${commit}" ]; then
     fi # otherwise, the repo already has a ref, so no need to create
 fi
 
+image_format=raw
+if [[ $image_type == qemu ]]; then
+    image_format=qcow2
+fi
 
-img=${name}-${build}-metal.raw
+img=${name}-${build}-${image_type}.${image_format}
 if [ -f "${builddir}/${img}" ]; then
-    echo "Bare metal image already exists"
+    echo "${image_type} image already exists"
     exit
 fi
 
 path=${PWD}/${img}
 
-echo "Estimating disk size..."
-/usr/lib/coreos-assembler/estimate-commit-disk-size --repo "$ostree_repo" "$ref" --add-percent 20 > "$PWD/tmp/ostree-size.json"
-size="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")"
-# extra size is the non-ostree partitions, see create_disk.sh
-size="$(( size + 513 ))M"
-echo "Disk size estimated to $size"
+# For bare metal images, we estimate the disk size. For qemu, we get it from
+# image.yaml.
+if [[ $image_type == metal ]]; then
+    echo "Estimating disk size..."
+    /usr/lib/coreos-assembler/estimate-commit-disk-size --repo "$ostree_repo" "$ref" --add-percent 20 > "$PWD/tmp/ostree-size.json"
+    size="$(jq '."estimate-mb".final' "$PWD/tmp/ostree-size.json")"
+    # extra size is the non-ostree partitions, see create_disk.sh
+    size="$(( size + 513 ))M"
+    echo "Disk size estimated to $size"
+else
+    size="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin)["size"])' < "$configdir/image.yaml")G"
+fi
+
 kargs="$(python3 -c 'import sys, yaml; args = yaml.safe_load(sys.stdin).get("extra-kargs", []); print(" ".join(args))' < "$configdir/image.yaml")"
 tty="console=tty0 console=${VM_TERMINAL},115200n8"
 # tty0 does not exist on s390x
 if [ "$arch" == "s390x" ]; then
     tty="console=${VM_TERMINAL}"
 fi
-kargs="$kargs $tty ignition.platform.id=metal"
+kargs="$kargs $tty ignition.platform.id=$image_type"
+
 ostree_remote="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin).get("ostree-remote", "NONE"))' < "$configdir/image.yaml")"
 save_var_subdirs="$(python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin).get("save-var-subdirs-for-selabel-workaround", "NONE"))' < "$configdir/image.yaml")"
 
-qemu-img create -f raw "${path}.tmp" "$size"
-runvm_with_disk "${path}.tmp" raw /usr/lib/coreos-assembler/create_disk.sh /dev/vda "$ostree_repo" "${ref-:${commit}}" "${ostree_remote}" /usr/lib/coreos-assembler/grub.cfg "$name" "${save_var_subdirs}" "\"$kargs\""
-mv "${path}.tmp" "$path"
-
-# flush it out before going to the next one so we don't have to redo both if
-# the next one fails
+if [ -z "${use_anaconda}" ]; then
+    qemu-img create -f ${image_format} "${path}.tmp" "$size"
+    runvm_with_disk "${path}.tmp" ${image_format} /usr/lib/coreos-assembler/create_disk.sh /dev/vda "$ostree_repo" "${ref-:${commit}}" "${ostree_remote}" /usr/lib/coreos-assembler/grub.cfg "$name" "${save_var_subdirs}" "\"$kargs\""
+    mv "${path}.tmp" "$path"
+    echo "{}" > tmp/vm-iso-checksum.json
+else
+    [ "${image_type}" == qemu ]
+    mkdir -p tmp/anaconda
+    # forgive me for this sin
+    checksum_location=$(find /usr/lib/coreos-assembler-anaconda/ -name '*CHECKSUM' | head -1)
+    img_base=tmp/${name}-${build}-base.qcow2
+    run_virtinstall "${ostree_repo}" "${ref}" "${PWD}"/"${img_base}" --variant=cloud
+    /usr/lib/coreos-assembler/gf-platformid "$(pwd)"/"${img_base}" "${path}" qemu
+    vm_iso_checksum=$(awk '/SHA256.*iso/{print$NF}' "${checksum_location}")
+    cat > tmp/vm-iso-checksum.json <<EOF
+{
+    "coreos-assembler.vm-iso-checksum": "${vm_iso_checksum}"
+}
+EOF
+fi
 
 # there's probably a jq one-liner for this...
 python3 -c "
 import sys, json
 j = json.load(sys.stdin)
-j['images']['metal'] = {
+j['images']['${image_type}'] = {
     'path': '${img}',
     'sha256': '$(sha256sum_str < "${img}")',
     'size': $(stat -c '%s' "${img}")
 }
 json.dump(j, sys.stdout, indent=4)
-" < "${builddir}/meta.json" > meta.json.new
+" < "${builddir}/meta.json" | cat - tmp/vm-iso-checksum.json | jq -s add > meta.json.new
 
 # and now the crucial bit
 mv -T meta.json.new "${builddir}/meta.json"

src/cmd-buildextend-qemu

@@ -0,0 +1 @@
+cmd-buildextend-metal

src/coreos-assembler

@@ -38,7 +38,7 @@ cmd=${1:-}
 build_commands="init fetch build run prune clean"
 # commands more likely to be used in a prod pipeline only
 advanced_build_commands="buildprep buildupload oscontainer"
-buildextend_commands="aws azure gcp openstack installer vmware metal"
+buildextend_commands="qemu aws azure gcp openstack installer vmware metal"
 utility_commands="tag compress bump-timestamp koji-upload kola aws-replicate"
 other_commands="shell"
 if [ -z "${cmd}" ]; then