cmd-build

#!/usr/bin/env bash
set -euo pipefail

dn=$(dirname "$0")
# shellcheck source=src/cmdlib.sh
. "${dn}"/cmdlib.sh

print_help() {
    cat 1>&2 <<'EOF'
Usage: coreos-assembler build --help
       coreos-assembler build [OPTIONS]... [TARGET]...

  Build OSTree and image base artifacts from previously fetched packages.
  Accepted TARGET arguments:

  - ostree     Compose an ostree commit
  - qemu       Also create a QCOW2 image to run with QEMU
  - metal      Also create a raw disk image
  - metal4k    Also create a raw disk image for 4K native disks

  The "qemu" and "metal" targets imply "ostree". If unspecified, defaults to
  "qemu". They are equivalent to manually running buildextend-[TARGET] after.

  The following options are supported:

  --delay-meta-merge  Set 'coreos-assembler.delayed-meta-merge' in build metadata (default: false)
  --force             Always create a new OSTree commit, even if nothing appears to have changed
  --force-image       Force an image rebuild even if there were no changes to image input
  --skip-prune        Skip prunning previous builds
  --strict            Only allow installing locked packages when using lockfiles
  --tag TAG           Set the given tag in the build metadata
  --version VERSION   Use the given version instead of generating one based on current time

  Additional environment variables supported:

  RPMOSTREE_EXTRA_ARGS         To pass extra arguments to 'rpm-ostree compose tree ...'
  RPMOSTREE_PRESERVE_TMPDIR    To keep the temporary compose rootfs from 'rpm-ostree compose tree ...'

EOF
}

# Parse options
DELAY_META_MERGE=false
FORCE=
FORCE_IMAGE=
SKIP_PRUNE=0
VERSION=
PARENT=
PARENT_BUILD=
TAG=
STRICT=
rc=0
options=$(getopt --options hft: --longoptions tag:,help,force,version:,parent:,parent-build:,delay-meta-merge,force-nocache,force-image,skip-prune,strict -- "$@") || rc=$?
[ $rc -eq 0 ] || {
    print_help
    exit 1
}
eval set -- "$options"
while true; do
    case "$1" in
        -h | --help)
            print_help
            exit 0
            ;;
        -f | --force | --force-nocache)
            FORCE="--force-nocache"
            ;;
        --delay-meta-merge)
            DELAY_META_MERGE=true
            ;;
        --force-image)
            FORCE_IMAGE=1
            ;;
        --skip-prune)
            SKIP_PRUNE=1
            ;;
        --strict)
            STRICT=1
            ;;
        --version)
            shift
            VERSION=$1
            ;;
        # XXX: to remove once --parent-build is plumbed through
        --parent)
            shift
            PARENT=$1
            ;;
        --parent-build)
            shift
            PARENT_BUILD=$1
            ;;
        -t | --tag)
            shift
            TAG=$1
            ;;
        --)
            shift
            break
            ;;
        -*)
            fatal "$0: unrecognized option: $1"
            exit 1
            ;;
        *)
            break
            ;;
    esac
    shift
done

if [ $# -eq 0 ]; then
    set -- qemu
fi

# sanity check the targets and aggregate into a set
declare -A targets=( )
for target in "$@"; do
    if [[ $target != ostree ]]; then
        case "$target" in
            metal|metal4k|qemu) ;;
            *) fatal "Unrecognized target: $target" ;;
        esac
        targets[$target]=1
    fi
done

build_followup_targets() {
    cd "${workdir}"
    for target in "${!targets[@]}"; do
        if ! "/usr/lib/coreos-assembler/cmd-buildextend-${target}"; then
            fatal "failed buildextend-${target}"
        fi
    done
}

prepare_build

ostree --version
rpm-ostree --version

previous_build=$(get_latest_build)
if [ -n "${previous_build}" ]; then
    previous_builddir=$(get_build_dir "${previous_build}")
fi
echo "Previous build: ${previous_build:-none}"

previous_commit=
previous_ostree_tarfile_path=
if [ -n "${previous_build}" ]; then
    previous_commit=$(jq -r '.["ostree-commit"]' < "${previous_builddir}/meta.json")
    previous_ostree_tarfile_path=$(jq -re '.images.ostree.path' < "${previous_builddir}/meta.json")
fi
echo "Previous commit: ${previous_commit:-none}"

if [ -n "${previous_commit}" ]; then
    # If we don't have the previous commit (or it's partial), then try to
    # re-import it; this saves us recompression time later on since it's likely
    # a lot of the new objects in this build will be the same.
    commitpath=${tmprepo}/objects/${previous_commit::2}/${previous_commit:2}.commit
    commitpartial=${tmprepo}/state/${previous_commit}.commitpartial
    if [ ! -f "${commitpath}" ] || [ -f "${commitpartial}" ]; then
        if [ -f "${previous_builddir}/${previous_ostree_tarfile_path}" ]; then
            import_ostree_commit_for_build "${previous_build}"
        else
            # ok, just fallback to importing the commit object only
            mkdir -p "$(dirname "${commitpath}")"
            cp "${previous_builddir}/ostree-commit-object" "${commitpath}"
            touch "${commitpartial}"
        fi
    fi

    # and point the ref to it if there isn't one already (in which case it might be newer, but e.g. creating disk failed)
    if test -n "${ref}" && ! ostree rev-parse --repo="${tmprepo}" "${ref}" &>/dev/null; then
        ostree refs --repo="${tmprepo}" --create "${ref}" "${previous_commit}"
    fi

    # Corner-case here: if the previous build was for a different ref, then we
    # want to make sure rpm-ostree doesn't select the same version. Do this by
    # pretending the ref is currently pointing at the last commit on the
    # previous ref. This is a dirty hack, though note all of cosa today knows to
    # not trust tmp/repo and only use it as an optimization. This corner-case is
    # also only relevant to developer workflows.
    previous_ref=$(jq -r '.["ref"]' < "${previous_builddir}/meta.json")
    if [ "${previous_ref}" != "null" ] && [ "${previous_ref}" != "${ref}" ]; then
        ostree reset --repo="${tmprepo}" "${ref}" "${previous_ref}"
        FORCE=--force-nocache
    fi
fi

if [ -n "${PARENT_BUILD}" ]; then
    parent_builddir=$(get_build_dir "${PARENT_BUILD}")
    PARENT=$(jq -r '.["ostree-commit"]' < "${parent_builddir}/meta.json")
    # and copy the parent into the repo so that we can generate a pkgdiff below
    commitpath=${tmprepo}/objects/${PARENT::2}/${PARENT:2}.commit
    commitpartial=${tmprepo}/state/${PARENT}.commitpartial
    mkdir -p "$(dirname "${commitpath}")" "$(dirname "${commitpartial}")"
    if [ ! -f "${commitpath}" ]; then
        cp "${parent_builddir}/ostree-commit-object" "${commitpath}"
        # and mark as partial since we only imported the commit object
        touch "${commitpartial}"
    fi
fi

# Calculate image input checksum now and gather previous image build variables if any
ks_path="${configdir}"/image.ks
if [ -f "${ks_path}" ]; then
    fatal "Kickstart support was removed; migrate to image.yaml"
fi
image_input="${image_yaml}"
if ! [ -f "${image_input}" ]; then
    fatal "Failed to find ${image_input}"
fi
image_config_checksum=$(< "${image_input}" sha256sum_str)
if [ -n "${previous_build}" ]; then
    previous_image_input_checksum=$(jq -r '.["coreos-assembler.image-input-checksum"]' < "${previous_builddir}/meta.json")
fi
echo "Image Config checksum: ${image_config_checksum}"

# Generate metadata that's *input* to the ostree commit
config_gitrev=$(cd "${configdir}" && git describe --tags --always --abbrev=42)
config_dirty=false
if ! git -C "${configdir}" diff --quiet --exit-code; then
    config_dirty=true
fi
commitmeta_input_json=${PWD}/tmp/commit-metadata-input.json
cat > "${commitmeta_input_json}" <<EOF
{
  "coreos-assembler.config-gitrev": "${config_gitrev}",
  "coreos-assembler.config-dirty": "${config_dirty}",
  "coreos-assembler.basearch": "${basearch}"
}
EOF

prepare_git_artifacts "${configdir_gitrepo}" "${PWD}/coreos-assembler-config.tar.gz" "${PWD}/coreos-assembler-config-git.json"

extra_compose_args=()

for lock in "${manifest_lock}" "${manifest_lock_overrides}" "${manifest_lock_arch_overrides}"; do
    if [ -f "${lock}" ]; then
        extra_compose_args+=("--ex-lockfile=${lock}")
    fi
done

if [ -n "${STRICT}" ]; then
    extra_compose_args+=("--ex-lockfile-strict")
fi

# We'll pass this directly to rpm-ostree instead of through
# commitmeta_input_json since that one also gets injected into meta.json, where
# there's already ostree-version.
if [ -n "${VERSION}" ]; then
    extra_compose_args+=("--add-metadata-string=version=${VERSION}")
fi

# Builds are independent of each other. Higher-level pipelines may want to force
# a specific parent, but otherwise we default to none. This is completely
# separate from pkg diffing, change detection, etc.
parent_arg=--no-parent
if [ -n "${PARENT}" ]; then
    parent_arg="--parent=${PARENT}"
fi
extra_compose_args+=("$parent_arg")

# These need to be absolute paths right now for rpm-ostree
composejson="$(readlink -f "${workdir}"/tmp/compose.json)"
# Put this under tmprepo so it gets automatically chown'ed if needed
lockfile_out=${tmprepo}/tmp/manifest-lock.generated.${basearch}.json
# shellcheck disable=SC2119
prepare_compose_overlays
# See https://github.com/coreos/coreos-assembler/pull/1379 - we want the local
# dev case to explicitly fetch updates when they want them, plus CI pipelines
# generally want to react to "changed or not" with a separate `fetch`.
# The fetched-stamp is new, in order to not break existing workdirs we assume
# a fetch was done if a successful build was done.
if [ ! -f "${workdir}"/builds/builds.json ] && [ ! -f "${fetch_stamp}" ] ; then
    fatal "Must fetch before building"
fi
# --cache-only is here since `fetch` is a separate verb
# shellcheck disable=SC2086
if test -n "${previous_commit}"; then
    extra_compose_args+=(--previous-commit "${previous_commit}")
fi
RUNVM_NONET=1 runcompose_tree --cache-only ${FORCE} \
           --add-metadata-from-json "${commitmeta_input_json}" \
           --write-composejson-to "${composejson}" \
           --ex-write-lockfile-to "${lockfile_out}".tmp \
           "${extra_compose_args[@]}"
strip_out_lockfile_digests "$lockfile_out".tmp
/usr/lib/coreos-assembler/finalize-artifact "${lockfile_out}"{.tmp,}
# Very special handling for --write-composejson-to as rpm-ostree doesn't
# write it if the commit didn't change.
if [ -f "${changed_stamp}" ] && [ -f "${composejson}" ]; then
    commit=$(jq -r '.["ostree-commit"]' < "${composejson}")
    # Clean up prior versions
    rm -f "${workdir}"/tmp/compose-*.json
    # Save this in case the image build fails
    cp-reflink "${composejson}" "${workdir}"/tmp/compose-"${commit}".json
else
    commit="${previous_commit}"
    image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str)
    echo "commit: ${commit} image: ${image_input_checksum}"
    # Note we may not actually have a previous build in the case of
    # successfully composing an ostree but failing the image on the
    # first build.
    # FORCE_IMAGE forces a build even if there were no ostree changes
    if [ -z "$FORCE_IMAGE" ] && [ -n "${previous_build}" ] && [ "${image_input_checksum}" = "${previous_image_input_checksum}" ]; then
        echo "No changes in image inputs."
        # But still run through the follow-up targets. This allows us to have
        # e.g. `cosa build metal` be idempotent even if the initial build failed
        # for whatever reason. `buildextend-[metal|qemu]` should already be
        # idempotent.
        build_followup_targets
        exit 0
    fi

    # Grab the previous treecompose JSON (local developer case: treecompose succeeded but
    # image build failed) if possible, otherwise grab the previous build
    cached_previous_composejson=${workdir}/tmp/compose-${commit}.json
    if [ -f "${cached_previous_composejson}" ]; then
        echo "Resuming partial build from: ${commit}"
        cp-reflink "${cached_previous_composejson}" "${composejson}"
    else
        if [ -z "${previous_build}" ]; then
            # This can happen if building the OSTree worked on the first time,
            # but image creation failed, and then tmp/ was nuked before trying a
            # second time. Just recommend re-running with --force.
            fatal "compose tree had no changes, but no previous build or cached data; try rerunning with --force"
        fi
        echo "Commit ${commit} unchanged; reusing previous build's rpm-ostree metadata"
        # This will have all of the data from the previous build, but we'll
        # overwrite things.
        cp-reflink "${previous_builddir}"/meta.json "${composejson}"
    fi
fi

if [ -n "${previous_build}" ]; then
    # do it once for the terminal
    rpm-ostree --repo="${tmprepo}" db diff --advisories "${previous_commit}" "${commit}"
    # and once more for the metadata, but only keep the pkgdiff and advisories keys
    rpm-ostree --repo="${tmprepo}" db diff --advisories --format=json \
            "${previous_commit}" "${commit}" | \
        jq '{"pkgdiff": .pkgdiff, "advisories-diff": .advisories}' > tmp/diff.json
else
    echo '{}' > tmp/diff.json
fi

if [ -n "${PARENT_BUILD}" ] && [[ ${PARENT} != "${previous_commit}" ]]; then
    rpm-ostree --repo="${tmprepo}" db diff --advisories --format=json \
            "${PARENT}" "${commit}" | \
        jq '{"parent-pkgdiff": .pkgdiff, "parent-advisories-diff": .advisories}' > tmp/parent-diff.json
else
    echo '{}' > tmp/parent-diff.json
fi

image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str)
echo "New image input checksum: ${image_input_checksum}"
init_build_meta_json "${commit}" "${PARENT_BUILD:-}" tmp/
buildid=$(jq -r '.["buildid"]' < tmp/meta.json)
echo "New build ID: ${buildid}"
# Also write out a ref with the build ID
ostree --repo="${tmprepo}" refs --create "${buildid}" "${commit}"

"${dn}"/write-commit-object "${tmprepo}" "${commit}" "$(pwd)"

build_timestamp=$(date -u +$RFC3339)

src_location="container"
if [ ! -f /lib/coreos-assembler/.clean ]; then
    info "This version of coreos-assembler is running code from outside the container."
    src_location="bind mount"
fi

# And create the ostree repo export containing the commit
ostree_tarfile_sha256=
if [ "${commit}" == "${previous_commit}" ] && \
    [ -f "${previous_builddir}/${previous_ostree_tarfile_path}" ]; then
    ostree_tarfile_path=$(jq -r '.images.ostree.path' < "${previous_builddir}/meta.json")
    cp-reflink "${previous_builddir}/${previous_ostree_tarfile_path}" "${ostree_tarfile_path}"
    ostree_tarfile_sha256=$(jq -r '.images.ostree.sha256' < "${previous_builddir}/meta.json")
    # backcompat: allow older build without this field
    if [ "${ostree_tarfile_sha256}" = "null" ]; then
        ostree_tarfile_sha256=
    fi
else
    ostree_format=$(jq -r '.["ostree-format"]' < "${image_json}")
    case "${ostree_format}" in
        null|tar)
            ostree_tarfile_path=${name}-${buildid}-ostree.${basearch}.tar
            ostree init --repo=repo --mode=archive
            # Pass the ref if it's set
            # shellcheck disable=SC2086
            if ! ostree pull-local --repo=repo "${tmprepo}" "${buildid}" ${ref}; then
                echo '(maybe https://github.com/coreos/coreos-assembler/issues/972 ?)'
                exit 1
            fi
            # Don't compress; archive repos are already compressed individually and we'd
            # gain ~20M at best. We could probably have better gains if we compress the
            # whole repo in bare/bare-user mode, but that's a different story...
            tar -cf "${ostree_tarfile_path}".tmp -C repo .
            rm -rf repo
            ;;
        oci)
            ostree_tarfile_path="${name}-${buildid}-ostree.${basearch}.ociarchive"
            rpm-ostree ex-container 'export' --cmd /usr/bin/bash --repo="${tmprepo}" "${buildid}" oci-archive:"${ostree_tarfile_path}".tmp:latest
            ;;
        *) fatal "Unknown ostree-format: ${ostree_format}"
    esac
    /usr/lib/coreos-assembler/finalize-artifact "${ostree_tarfile_path}"{.tmp,}
    ostree_tarfile_sha256=$(sha256sum "${ostree_tarfile_path}" | awk '{print$1}')
fi

# The base metadata, plus locations for code sources.
# If the following condition is true, then /lib/coreos-assembler has been bind
# mounted in and is using a different build tree.
#
# notice need to backslash escape double quotes in summary since the
# summary could have double quotes: https://github.com/coreos/coreos-assembler/issues/327
#
# shellcheck disable=SC2046 disable=SC2086
cat > tmp/buildmeta.json <<EOF
{
 "name": "${name}",
 "summary": "${summary//\"/\\\"}",
 "coreos-assembler.build-timestamp": "${build_timestamp}",
 "coreos-assembler.image-config-checksum": "${image_config_checksum}",
 "coreos-assembler.image-input-checksum": "${image_input_checksum}",
 "coreos-assembler.code-source": "${src_location}",
 "coreos-assembler.container-config-git": $(jq -M '.git' ${PWD}/coreos-assembler-config-git.json),
 "coreos-assembler.meta-stamp": $(python3 -c 'import time; print(time.time_ns())'),
 "coreos-assembler.delayed-meta-merge": ${DELAY_META_MERGE},
 "coreos-assembler.meta-stamp": $(date +%s%9N)
}
EOF

ostree_tarfile_size=$(stat --format=%s "${ostree_tarfile_path}")
cat > tmp/images.json <<EOF
{
  "images": {
    "ostree": {
        "path": "${ostree_tarfile_path}",
        "sha256": "${ostree_tarfile_sha256}",
        "size": ${ostree_tarfile_size}
    }
  }
}
EOF

overridesjson=tmp/overrides.json
if [ -f "${overrides_active_stamp}" ]; then
    echo '{ "coreos-assembler.overrides-active": true }' > "${overridesjson}"
else
    echo '{}' > "${overridesjson}"
fi

# And the build information about our container, if we are executing
# from a container.
if [ -d /cosa ]; then
    cat > tmp/cosa-image.json <<EOF
{ "coreos-assembler.container-image-git": $(jq -M '.git' /cosa/coreos-assembler-git.json) }
EOF
else
    echo '{}' > tmp/cosa-image.json
fi

# Merge all the JSON; note that we want ${composejson} first
# since we may be overriding data from a previous build.
cat "${composejson}" "${overridesjson}" tmp/meta.json tmp/buildmeta.json tmp/diff.json tmp/parent-diff.json tmp/images.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json

# Move lockfile into build dir
mv "${lockfile_out}" .

# And add the commit metadata itself, which includes notably the rpmdb pkglist
# in a format that'd be easy to generate diffs out of for higher level tools
"${dn}"/commitmeta_to_json "${tmprepo}" "${commit}" > commitmeta.json.tmp
/usr/lib/coreos-assembler/finalize-artifact commitmeta.json{.tmp,}

# Clean up our temporary data
saved_build_tmpdir="${workdir}/tmp/last-build-tmp"
rm -rf "${saved_build_tmpdir}"
mv -T tmp "${saved_build_tmpdir}"
# just keep the last 3 commits as a rough guideline; this matches
# DEFAULT_KEEP_LAST_N in `cmd-prune`
ostree prune --repo="${tmprepo}" --refs-only --depth=2
# Back to the toplevel work directory, so we can rename this one
cd "${workdir}"
# We create a .build-commit file to note that we're in the
# middle of a "commit".  This may be useful in the future
# for having things be transactional.  If for example we
# were interrupted between the rename() and linkat() below,
# things would be inconsistent and future builds would fail
# on the `mv`.
touch builds/.build-commit
builddir=$(get_build_dir "${buildid}")
mkdir -p "${builddir}"
mv -T "${tmp_builddir}" "${builddir}"
# Replace the latest link
ln -Tsf "${buildid}" builds/latest

if [ "${SKIP_PRUNE}" == 1 ]; then
  insert_build "${buildid}" "${workdir}"
else
  "${dn}"/cmd-prune --workdir "${workdir}"
fi
rm builds/.build-commit

if [ -n "${TAG}" ]; then
    # ideally, we'd do this atomically before moving to builds/latest, but
    # meh... not much can go wrong with `cosa tag`
    /usr/lib/coreos-assembler/cmd-tag update --build "${buildid}" --tag "${TAG}"
fi

# and finally, build the specified targets
build_followup_targets