.gitlab-ci-check-docker-build.yml

# .gitlab-ci-check-docker-build.yml
#
# This gitlab-ci template builds a Docker image and publishes it
# on Docker Hub or Mender Registry
#
# Requires DOCKER_REPOSITORY variable to be set in the calling Pipeline.
# Add it to the project in hand through Gitlab's include functionality
# variables:
#   DOCKER_REPOSITORY: <Image FQN, i.e mendersoftware/reponame>
# include:
#   - project: 'Northern.tech/Mender/mendertesting'
#     file: '.gitlab-ci-check-docker-build.yml'
#
# It assumes a Dockerfile in the root of the repository. A different
# directory or a different Dockerfile filename can optionally be specified
# using the following variables:
#   DOCKER_DIR: <relative path, i.e service/> (optional)
#   DOCKERFILE: <filename, i.e. Dockerfile.custom> (optional)
#
# The Docker image is saved as job artifact in ${CI_PROJECT_DIR}/image.tar and
# loaded from there in further jobs. A different filename can be specified with:
#   IMAGE_ARTIFACT_FILENAME: <full path to Docker image filename> (optional)
#
# Requires credentials for the registry where to push the image.
# Set in the project CI/CD settings either Docker Hub credentials:
#   DOCKER_HUB_USERNAME: Username for docker.io
#   DOCKER_HUB_PASSWORD: Password for docker.io
# or Mender Registry ones:
#   REGISTRY_MENDER_IO_USERNAME: Username for registry.mender.io
#   REGISTRY_MENDER_IO_PASSWORD: Password for registry.mender.io
#
# Passes the following variables to the build:
#   * GIT_COMMIT_TAG
#     * SHA of git commit at which build ocurred
#     * passed via: --build-arg GIT_COMMIT_TAG="${DOCKER_PUBLISH_COMMIT_TAG}"
#   * DOCKER_BUILD_ARGS 
#     * Contains additional --build-arg parameters that should be passed to the build
#
# Saves in a dotenv file the published image digests. These env variables
# can be used by later jobs that have publish:image or publish:image:mender
# as dependencies:
#   * PUBLISH_IMAGE_DIGEST
#   * PUBLISH_IMAGE_MENDER_DIGEST
#
#
# Multiplafrom build:
# Excludes build:docker and publish:image.
#
# Required variables:
# MULTIPLATFORM_BUILD: "true"
# Configurable variables:
# MULTIPLATFORM_PLATFORMS: "linux/amd64,linux/armv7"
#
# Note: since jobs are evaluated before variables,
# you have to override the build:docker and publish:image: job on the parent .gitlab-ci.yml;
# otherwise you'll get both build:docker and build:docker-multiplatform jobs.
# E.g.:
#
# build:docker:
#   rules:
#     - when: never
#
# publish:image:
#   rules:
#     - when: never
#
#


variables:
  MULTIPLATFORM_PLATFORMS:
    description: "Comma-separated list of targets to build os/arch[/<version>]"
    value: "linux/amd64,linux/arm64"
  EXTRA_DOCKER_ARGS:
    description: "User specified extra arguments for docker build command"
    value: ""

stages:
  - build
  - publish

.docker_login_registries: &docker_login_registries |
  if [ -n "$REGISTRY_MENDER_IO_PASSWORD" ]; then \
    echo "${REGISTRY_MENDER_IO_PASSWORD}" | docker login -u "${REGISTRY_MENDER_IO_USERNAME}" --password-stdin registry.mender.io || \
    echo "Warning: registry.mender.io credentials invalid"; \
  else \
    echo "Warning: registry.mender.io credentials unavailable"; \
  fi; \
  if [ -n "$DOCKER_HUB_PASSWORD" ]; then \
    echo "${DOCKER_HUB_PASSWORD}" | docker login -u "${DOCKER_HUB_USERNAME}" --password-stdin || \
    echo "Warning: Docker Hub credentials invalid"; \
  else \
    echo "Warning: Docker Hub credentials unavailable"; \
  fi

.export_docker_vars: &export_docker_vars |
  DOCKER_BUILD_TAG=${CI_COMMIT_REF_SLUG:-local}
  DOCKER_BUILD_SERVICE_IMAGE=${DOCKER_REPOSITORY}:${DOCKER_BUILD_TAG}
  DOCKER_PUBLISH_TAG=${CI_COMMIT_REF_NAME}
  DOCKER_PUBLISH_COMMIT_TAG=${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}

.get_release_tool_alpine: &get_release_tool_alpine |
  apk add git python3 py3-virtualenv py3-pip
  git clone https://github.com/mendersoftware/integration.git mender-integration
  python3 -m virtualenv .venv
  source .venv/bin/activate
  pip3 install -r mender-integration/extra/requirements.txt
  alias release_tool=$(realpath mender-integration/extra/release_tool.py)

.dind-login: &dind-login
  - mkdir -p $HOME/.docker && echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json

build:docker:
  tags:
    - mender-qa-worker-generic-light
  stage: build
  needs: []
  variables:
    DOCKER_BUILDKIT: 1
  rules:
    - if: '$CI_COMMIT_TAG =~ /^saas-[a-zA-Z0-9.]+$/'
      when: never
    - when: on_success
  image: docker
  services:
    - docker:20.10.21-dind
  before_script:
    - *export_docker_vars
  script:
    - echo "building ${CI_PROJECT_NAME} for ${DOCKER_BUILD_SERVICE_IMAGE}"
    - docker build
        --tag $DOCKER_BUILD_SERVICE_IMAGE
        --file ${DOCKER_DIR:-.}/${DOCKERFILE:-Dockerfile}
        --build-arg GIT_COMMIT_TAG="${DOCKER_PUBLISH_COMMIT_TAG}"
        ${DOCKER_BUILD_ARGS}
        ${DOCKER_DIR:-.}
    - docker save $DOCKER_BUILD_SERVICE_IMAGE > ${IMAGE_ARTIFACT_FILENAME:-${CI_PROJECT_DIR}/image.tar}
  artifacts:
    expire_in: 2w
    paths:
      - ${IMAGE_ARTIFACT_FILENAME:-image.tar}

build:docker-multiplatform:
  tags:
    - hetzner-amd-beefy
  stage: build
  needs: []
  variables:
    DOCKER_BUILDKIT: 1
    GITLAB_REGISTRY_TAG: '${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID}'
  rules:
    - if: $MULTIPLATFORM_BUILD != "true"
      when: never
    - if: '$CI_COMMIT_TAG =~ /^saas-[a-zA-Z0-9.]+$/'
      when: never
    - when: on_success
  image: "registry.gitlab.com/northern.tech/mender/mender-test-containers:docker-multiplatform-buildx-v1-master"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:20.10.21-dind
      alias: docker
  before_script:
    - *dind-login
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - echo "building ${CI_PROJECT_NAME} with tags ${GITLAB_REGISTRY_TAG} and ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}"
    - docker context create builder
    - docker buildx create builder --use --driver-opt "image=moby/buildkit:v0.17.3,network=host" --buildkitd-flags '--debug --allow-insecure-entitlement network.host'
    - docker buildx build
      --cache-to type=registry,ref=${CI_REGISTRY_IMAGE}:ci_cache,mode=max
      --cache-from type=registry,ref=${CI_REGISTRY_IMAGE}:ci_cache
      --tag ${GITLAB_REGISTRY_TAG}
      --tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_NAME}
      --file ${DOCKER_DIR:-.}/${DOCKERFILE:-Dockerfile}
      --build-arg GIT_COMMIT_TAG="${DOCKER_PUBLISH_COMMIT_TAG}"
      --platform $MULTIPLATFORM_PLATFORMS
      --provenance false
      --push
      ${EXTRA_DOCKER_ARGS}
      ${DOCKER_DIR:-.}

publish:image:
  tags:
    - mender-qa-worker-generic-light
  stage: publish
  rules:
    - if: '$CI_COMMIT_BRANCH =~ /^(master|staging|production|feature-.+)$/'
  image: docker
  services:
    - docker:20.10.21-dind
  dependencies:
    - build:docker
  before_script:
    - *export_docker_vars
    - *docker_login_registries
  script:
    - docker load -i ${IMAGE_ARTIFACT_FILENAME:-${CI_PROJECT_DIR}/image.tar}
    - docker tag $DOCKER_BUILD_SERVICE_IMAGE $DOCKER_REPOSITORY:$DOCKER_PUBLISH_TAG
    - docker tag $DOCKER_BUILD_SERVICE_IMAGE $DOCKER_REPOSITORY:$DOCKER_PUBLISH_COMMIT_TAG
    - docker push $DOCKER_REPOSITORY:$DOCKER_PUBLISH_TAG
    - docker push $DOCKER_REPOSITORY:$DOCKER_PUBLISH_COMMIT_TAG
    - echo "PUBLISH_IMAGE_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:$DOCKER_PUBLISH_TAG)" >> publish.env
  artifacts:
    reports:
      dotenv: publish.env

publish:image-multiplatform:
  tags:
    - hetzner-amd-beefy
  stage: publish
  variables:
    GITLAB_REGISTRY_TAG: '${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID}'
  rules:
    - if: $MULTIPLATFORM_BUILD != "true"
      when: never
    - if: '$CI_COMMIT_BRANCH =~ /^(main|master|staging|production|feature-.+)$/'
  image: "registry.gitlab.com/northern.tech/mender/mender-test-containers:docker-multiplatform-buildx-v1-master"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:20.10.21-dind
      alias: docker
  dependencies:
    - build:docker-multiplatform
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - *export_docker_vars
    - *docker_login_registries
  script:
    - regctl image copy ${GITLAB_REGISTRY_TAG} $DOCKER_REPOSITORY:$DOCKER_PUBLISH_TAG
    - regctl image copy ${GITLAB_REGISTRY_TAG} $DOCKER_REPOSITORY:$DOCKER_PUBLISH_COMMIT_TAG
    - echo "PUBLISH_IMAGE_DIGEST=${DOCKER_REPOSITORY}@$(regctl image digest ${GITLAB_REGISTRY_TAG})" >> publish.env
  artifacts:
    reports:
      dotenv: publish.env


publish:image:mender:
  tags:
    - mender-qa-worker-generic-light
  stage: publish
  rules:
    - if: '$CI_COMMIT_BRANCH =~ /^(master|[0-9]+\.[0-9]+\.x)$/'
  image: docker
  services:
    - docker:20.10.21-dind
  dependencies:
    - build:docker
  before_script:
    # Use same variables for loading the image, while DOCKER_PUBLISH_COMMIT_TAG will be ignored
    - *export_docker_vars
    - *get_release_tool_alpine
    - *docker_login_registries
    # If the repo is not recognized, ignore
    - if ! echo $(release_tool --list git --all) | grep $CI_PROJECT_NAME; then
    -  echo "Repository $CI_PROJECT_NAME not found in release_tool. Exiting"
    -  exit 137
    - fi
    # If the repo/branch is not part of a Mender release, also ignore
    - integration_versions=$(release_tool --integration-versions-including $CI_PROJECT_NAME --version $CI_COMMIT_REF_NAME | sed -e 's/origin\///')
    - if test -z "$integration_versions"; then
    -  echo "Repository $CI_PROJECT_NAME version $CI_COMMIT_REF_NAME is not part of any Mender release. Exiting"
    -  exit 137
    - fi
  allow_failure:
    exit_codes: 137
  script:
    # Load image
    - docker load -i ${IMAGE_ARTIFACT_FILENAME:-${CI_PROJECT_DIR}/image.tar}
    # Publish the image for all releases
    - for version in $integration_versions; do
    -   docker tag $DOCKER_BUILD_SERVICE_IMAGE $DOCKER_REPOSITORY:mender-${version}
    -   docker tag $DOCKER_BUILD_SERVICE_IMAGE $DOCKER_REPOSITORY:mender-${version}_${CI_COMMIT_SHA}
    -   docker push $DOCKER_REPOSITORY:mender-${version}
    -   docker push $DOCKER_REPOSITORY:mender-${version}_${CI_COMMIT_SHA}
    - done
    - echo "PUBLISH_IMAGE_MENDER_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $DOCKER_REPOSITORY:mender-${version})" >> publish.env
  artifacts:
    reports:
      dotenv: publish.env

publish:image-multiplatform:mender:
  tags:
    - hetzner-amd-beefy
  stage: publish
  variables:
    GITLAB_REGISTRY_TAG: '${CI_REGISTRY_IMAGE}:${CI_PIPELINE_ID}'
  rules:
    - if: $MULTIPLATFORM_BUILD != "true"
      when: never
    - if: '$CI_COMMIT_BRANCH =~ /^(master|[0-9]+\.[0-9]+\.x)$/'
  image: "registry.gitlab.com/northern.tech/mender/mender-test-containers:docker-multiplatform-buildx-v1-master"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:20.10.21-dind
      alias: docker
  dependencies:
    - build:docker-multiplatform
  before_script:
    # Use same variables for loading the image, while DOCKER_PUBLISH_COMMIT_TAG will be ignored
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - *export_docker_vars
    - *get_release_tool_alpine
    - *docker_login_registries
    # If the repo is not recognized, ignore
    - if ! echo $(release_tool --list git --all) | grep $CI_PROJECT_NAME; then
    -  echo "Repository $CI_PROJECT_NAME not found in release_tool. Exiting"
    -  exit 137
    - fi
    # If the repo/branch is not part of a Mender release, also ignore
    - integration_versions=$(release_tool --integration-versions-including $CI_PROJECT_NAME --version $CI_COMMIT_REF_NAME | sed -e 's/origin\///')
    - if test -z "$integration_versions"; then
    -  echo "Repository $CI_PROJECT_NAME version $CI_COMMIT_REF_NAME is not part of any Mender release. Exiting"
    -  exit 137
    - fi
  allow_failure:
    exit_codes: 137
  script:
    # Publish the image for all releases
    - for version in $integration_versions; do
    -   regctl image copy ${GITLAB_REGISTRY_TAG} ${DOCKER_REPOSITORY}:mender-${version}
    -   regctl image copy ${GITLAB_REGISTRY_TAG} ${DOCKER_REPOSITORY}:mender-${version}_${CI_COMMIT_SHA}
    - done
    - echo "PUBLISH_IMAGE_DIGEST=${DOCKER_REPOSITORY}@$(regctl image digest $DOCKER_REPOSITORY:mender-${version})" >> publish.env
  artifacts:
    reports:
      dotenv: publish.env

# saas-specific job to retag docker images after saas-* tags are pushed to the repository
publish:image:saas:
  tags:
    - mender-qa-worker-generic-light
  stage: publish
  rules:
    - if: '$CI_COMMIT_TAG =~ /^saas-[a-zA-Z0-9.]+$/'
  image: docker
  services:
    - docker:20.10.21-dind
  before_script:
    - SOURCE_TAG=staging_${CI_COMMIT_SHA}
    - DOCKER_PUBLISH_TAG=${CI_COMMIT_REF_NAME}
    - SERVICE_IMAGE=${DOCKER_REPOSITORY}:${DOCKER_PUBLISH_TAG}
    - *docker_login_registries
  script:
    - docker pull $DOCKER_REPOSITORY:$SOURCE_TAG
    - docker tag $DOCKER_REPOSITORY:$SOURCE_TAG $SERVICE_IMAGE
    - docker push $SERVICE_IMAGE

publish:image-multiplatform:saas:
  tags:
    - hetzner-amd-beefy
  stage: publish
  rules:
    - if: $MULTIPLATFORM_BUILD != "true"
      when: never
    - if: '$CI_COMMIT_TAG =~ /^saas-[a-zA-Z0-9.]+$/'
  image: "registry.gitlab.com/northern.tech/mender/mender-test-containers:docker-multiplatform-buildx-v1-master"
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:20.10.21-dind
      alias: docker
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - *export_docker_vars
    - *docker_login_registries
    - SOURCE_TAG=staging_${CI_COMMIT_SHA}
    - DOCKER_PUBLISH_TAG=${CI_COMMIT_REF_NAME}
    - SERVICE_IMAGE=${DOCKER_REPOSITORY}:${DOCKER_PUBLISH_TAG}
  script:
    - regctl manifest get ${DOCKER_REPOSITORY}:${SOURCE_TAG} || (echo "ERROR - image ${DOCKER_REPOSITORY}:${SOURCE_TAG} not found"; exit 1)
    - regctl image copy ${DOCKER_REPOSITORY}:${SOURCE_TAG} ${SERVICE_IMAGE}