diff --git a/scripts/github-actions/example-ci.yml b/scripts/github-actions/example-ci.yml new file mode 100644 index 00000000..155ef316 --- /dev/null +++ b/scripts/github-actions/example-ci.yml @@ -0,0 +1,48 @@ +name: GitHub CI + +on: + pull_request: + push: + schedule: + - cron: 0 0 * * 0 + +defaults: + run: + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + + generate-jobs: + name: Generate Jobs + runs-on: ubuntu-latest + outputs: + strategy: ${{ steps.generate-jobs.outputs.strategy }} + steps: + - uses: actions/checkout@v1 + - id: generate-jobs + name: Generate Jobs + run: | + git clone --depth 1 https://github.com/docker-library/bashbrew.git -b master ~/bashbrew + strategy="$(~/bashbrew/scripts/github-actions/generate.sh)" + jq . <<<"$strategy" # sanity check / debugging aid + echo "::set-output name=strategy::$strategy" + + test: + needs: generate-jobs + strategy: ${{ fromJson(needs.generate-jobs.outputs.strategy) }} + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v1 + - name: Prepare Environment + run: ${{ matrix.runs.prepare }} + - name: Pull Dependencies + run: ${{ matrix.runs.pull }} + - name: Build ${{ matrix.name }} + run: ${{ matrix.runs.build }} + - name: History ${{ matrix.name }} + run: ${{ matrix.runs.history }} + - name: Test ${{ matrix.name }} + run: ${{ matrix.runs.test }} + - name: '"docker images"' + run: ${{ matrix.runs.images }} diff --git a/scripts/github-actions/generate.sh b/scripts/github-actions/generate.sh new file mode 100755 index 00000000..9ee824b9 --- /dev/null +++ b/scripts/github-actions/generate.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +image="${GITHUB_REPOSITORY##*/}" # "python", "golang", etc + +[ -n "${GENERATE_STACKBREW_LIBRARY:-}" ] || [ -x ./generate-stackbrew-library.sh ] # sanity check + +tmp="$(mktemp -d)" +trap "$(printf 'rm -rf %q' "$tmp")" EXIT + +if ! command -v bashbrew &> /dev/null; then + dir="$(readlink -f "$BASH_SOURCE")" + dir="$(dirname "$dir")" + dir="$(cd "$dir/../.." && pwd -P)" + if [ ! -x "$dir/bin/bashbrew" ]; then + echo >&2 'Building bashbrew ...' + "$dir/bashbrew.sh" --version > /dev/null + "$dir/bin/bashbrew" --version >&2 + fi + export PATH="$dir/bin:$PATH" + bashbrew --version > /dev/null +fi + +mkdir "$tmp/library" +export BASHBREW_LIBRARY="$tmp/library" + +eval "${GENERATE_STACKBREW_LIBRARY:-./generate-stackbrew-library.sh}" > "$BASHBREW_LIBRARY/$image" + +tags="$(bashbrew list --build-order --uniq "$image")" + +# see https://github.com/docker-library/python/commit/6b513483afccbfe23520b1f788978913e025120a for the ideal of what this would be (minimal YAML in all 30+ repos, shared shell script that outputs fully dynamic steps list), if GitHub Actions were to support a fully dynamic steps list + +order=() +declare -A metas=() +for tag in $tags; do + echo >&2 "Processing $tag ..." + bashbrewImage="${tag##*/}" # account for BASHBREW_NAMESPACE being set + meta="$( + bashbrew cat --format ' + {{- $e := .TagEntry -}} + {{- "{" -}} + "name": {{- json ($e.Tags | first) -}}, + "tags": {{- json ($.Tags namespace false $e) -}}, + "directory": {{- json $e.Directory -}}, + "file": {{- json $e.File -}}, + "constraints": {{- json $e.Constraints -}}, + "froms": {{- json ($.DockerFroms $e) -}} + {{- "}" -}} + ' "$bashbrewImage" | jq -c ' + { + name: .name, + os: ( + if (.constraints | contains(["windowsservercore-1809"])) or (.constraints | contains(["nanoserver-1809"])) then + "windows-2019" + elif .constraints | contains(["windowsservercore-ltsc2016"]) then + "windows-2016" + elif .constraints == [] or .constraints == ["!aufs"] then + "ubuntu-latest" + else + # use an intentionally invalid value so that GitHub chokes and we notice something is wrong + "invalid-or-unknown" + end + ), + meta: { entries: [ . ] }, + runs: { + build: ( + [ + "docker build" + ] + + ( + .tags + | map( + "--tag " + (. | @sh) + ) + ) + + if .file != "Dockerfile" then + [ "--file", (.file | @sh) ] + else + [] + end + + [ + (.directory | @sh) + ] + | join(" ") + ), + history: ("docker history " + (.tags[0] | @sh)), + test: ("~/oi/test/run.sh " + (.tags[0] | @sh)), + }, + } + ' + )" + + parent="$(bashbrew parents "$bashbrewImage" | tail -1)" # if there ever exists an image with TWO parents in the same repo, this will break :) + if [ -n "$parent" ]; then + parentBashbrewImage="${parent##*/}" # account for BASHBREW_NAMESPACE being set + parent="$(bashbrew list --uniq "$parentBashbrewImage")" # normalize + parentMeta="${metas["$parent"]}" + parentMeta="$(jq -c --argjson meta "$meta" ' + . + { + name: (.name + ", " + $meta.name), + os: (if .os == $meta.os then .os else "invalid-os-chain--" + .os + "+" + $meta.os end), + meta: { entries: (.meta.entries + $meta.meta.entries) }, + runs: ( + .runs + | to_entries + | map( + .value += "\n" + $meta.runs[.key] + ) + | from_entries + ), + } + ' <<<"$parentMeta")" + metas["$parent"]="$parentMeta" + else + metas["$tag"]="$meta" + order+=( "$tag" ) + fi +done + +strategy="$( + for tag in "${order[@]}"; do + jq -c ' + .meta += { + froms: ( + [ .meta.entries[].froms[] ] + - [ .meta.entries[].tags[] ] + ), + dockerfiles: [ + .meta.entries[] + | .directory + "/" + .file + ], + } + | .runs += { + prepare: ([ + ( + if .os | startswith("windows-") then + "# enable symlinks on Windows (https://git-scm.com/docs/git-config#Documentation/git-config.txt-coresymlinks)", + "git config --global core.symlinks true", + "# ... make sure they are *real* symlinks (https://github.com/git-for-windows/git/pull/156)", + "export MSYS=winsymlinks:nativestrict", + "# make sure line endings get checked out as-is", + "git config --global core.autocrlf false" + else + empty + end + ), + "git clone --depth 1 https://github.com/docker-library/official-images.git -b master ~/oi", + "# create a dummy empty image/layer so we can --filter since= later to get a meanginful image list", + "{ echo FROM " + ( + if (.os | startswith("windows-")) then + "mcr.microsoft.com/windows/servercore:ltsc" + (.os | ltrimstr("windows-")) + else + "busybox:latest" + end + ) + "; echo RUN :; } | docker build --no-cache --tag image-list-marker -", + ( + if .os | startswith("windows-") | not then + ( + "# PGP Happy Eyeballs", + "git clone --depth 1 https://github.com/tianon/pgp-happy-eyeballs.git ~/phe", + "~/phe/hack-my-builds.sh", + "rm -rf ~/phe" + ) + else + empty + end + ) + ] | join("\n")), + pull: ([ .meta.froms[] | select(. != "scratch") | "docker pull " + @sh ] | join("\n")), + # build + # history + # test + images: "docker image ls --filter since=image-list-marker", + } + ' <<<"${metas["$tag"]}" + done | jq -cs ' + { + "fail-fast": false, + matrix: { include: . }, + } + ' +)" + +if [ -t 1 ]; then + jq <<<"$strategy" +else + cat <<<"$strategy" +fi diff --git a/scripts/github-actions/munge-i386.sh b/scripts/github-actions/munge-i386.sh new file mode 100755 index 00000000..2adf8798 --- /dev/null +++ b/scripts/github-actions/munge-i386.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +jq --arg dpkgSmokeTest '[ "$(dpkg --print-architecture)" = "amd64" ]' ' + .matrix.include += [ + .matrix.include[] + | select(.name | test(" (.+)") | not) # ignore any existing munged builds + | select(.os | startswith("windows-") | not) + | .name += " (i386)" + | .runs.pull = ([ + "# pull i386 variants of base images for multi-architecture testing", + $dpkgSmokeTest, + ( + .meta.froms[] + | ("i386/" + . | @sh) as $i386 + | ( + "docker pull " + $i386, + "docker tag " + $i386 + " " + @sh + ) + ) + ] | join("\n")) + ] +' "$@"