Update api version, vendor & enhance testing

.github/workflows/test.yml

@@ -24,17 +24,7 @@ jobs:
           sudo chown -R "$USER" "$HOME/.kube" "$HOME/.minikube"
           sudo usermod -aG docker "$USER"
       - run: |
-          mkdir -p certs
-          openssl req -x509 -newkey rsa:4096 -keyout certs/key.pem -out certs/cert.pem -days 365 -subj '/CN=localhost' -nodes -addext 'subjectAltName = DNS:localhost'
-          docker run -d --restart=always --name registry -v "$(pwd)"/certs:/certs -e REGISTRY_HTTP_ADDR=0.0.0.0:443 -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/cert.pem \
-                      -e REGISTRY_HTTP_TLS_KEY=/certs/key.pem \
-                      -p 443:443 \
-                      registry:2
-          sudo mkdir /etc/docker/certs.d
-          sudo mkdir /etc/docker/certs.d/localhost:443
-          sudo cp certs/cert.pem /etc/docker/certs.d/localhost:443/ca.crt
-          sudo cp certs/cert.pem /usr/local/share/ca-certificates/ca.crt
-          sudo update-ca-certificates
+          "${GITHUB_WORKSPACE}/scripts/start_registry.sh" minikube-registry
           export DOCKER_REGISTRY_HOST=localhost:443
       - run: |
           KUBECONFIG="$HOME/.kube/config" DOCKER_REGISTRY_HOST=localhost:443 make build e2e
@@ -47,26 +37,7 @@ jobs:
           sudo apt-get -y update
           podman version
       - run: |
-          curl -sLo kind "$(curl -sL https://api.github.com/repos/kubernetes-sigs/kind/releases/latest | jq -r '[.assets[] | select(.name == "kind-linux-amd64")] | first | .browser_download_url')"
-          chmod +x kind
-          sudo mv kind /bin/
-      - run: |
-          kind create cluster
-          kind export kubeconfig
-          sudo chown -R "$USER" "$HOME/.kube"
-          sudo usermod -aG docker "$USER"
-      - run: |
-          mkdir -p certs
-          openssl req -x509 -newkey rsa:4096 -keyout certs/key.pem -out certs/cert.pem -days 365 -subj '/CN=localhost' -nodes -addext 'subjectAltName = DNS:localhost'
-          docker run -d --restart=always --name registry -v "$(pwd)"/certs:/certs -e REGISTRY_HTTP_ADDR=0.0.0.0:443 -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/cert.pem \
-                      -e REGISTRY_HTTP_TLS_KEY=/certs/key.pem \
-                      -p 443:443 \
-                      registry:2
-          sudo mkdir /etc/docker/certs.d
-          sudo mkdir /etc/docker/certs.d/localhost:443
-          sudo cp certs/cert.pem /etc/docker/certs.d/localhost:443/ca.crt
-          sudo cp certs/cert.pem /usr/local/share/ca-certificates/ca.crt
-          sudo update-ca-certificates
+          "${GITHUB_WORKSPACE}/scripts/start_registry.sh" kind-registry
           export DOCKER_REGISTRY_HOST=localhost:443
       - run: |
-          KUBECONFIG="$HOME/.kube/config" DOCKER_REGISTRY_HOST=localhost:443 make build e2e
+          KUBECONFIG="$HOME/.kube/config" DOCKER_REGISTRY_HOST=localhost:443 make build e2e CLUSTER=kind

.gitignore

@@ -462,3 +462,13 @@ pkg/apprclient/openapi/git_push.sh
 !test/**testdata/**
 
 /coverage.out
+
+# Test artifacts generated by e2e
+test/e2e/bundle.Dockerfile
+test/e2e/downloaded/**
+test/e2e/opm-*
+test/e2e/bundle_tmp*
+test/e2e/index_tmp*
+
+# don't check in the certs directory
+certs/*

Makefile

@@ -6,15 +6,30 @@ PKG := github.com/operator-framework/operator-registry
 GIT_COMMIT := $(or $(SOURCE_GIT_COMMIT),$(shell git rev-parse --short HEAD))
 OPM_VERSION := $(or $(SOURCE_GIT_TAG),$(shell git describe --always --tags HEAD))
 BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
-TAGS := -tags "json1"
+
+# define characters
+null  :=
+space := $(null) #
+comma := ,
+# default to json1 for sqlite3
+TAGS := -tags=json1
+
+# Cluster to use for e2e testing 
+CLUSTER ?= ""
+ifeq ($(CLUSTER), kind)
+# add kind to the list of tags
+TAGS += kind
+# convert tag format from space to comma list
+TAGS := $(subst $(space),$(comma),$(strip $(TAGS)))
+endif
+
 # -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64
 ifeq ($(shell go env GOARCH),s390x)
 TEST_RACE :=
 else
 TEST_RACE := -race
 endif
 
-
 .PHONY: all
 all: clean test build
 
@@ -98,4 +113,4 @@ clean:
 
 .PHONY: e2e
 e2e:
-	$(GO) run github.com/onsi/ginkgo/ginkgo --v --randomizeAllSpecs --randomizeSuites --race $(TAGS) ./test/e2e
+	$(GO) run github.com/onsi/ginkgo/ginkgo --v --randomizeAllSpecs --randomizeSuites --race $(if $(TEST),-focus '$(TEST)') $(TAGS) ./test/e2e -- $(if $(SKIPTLS),-skip-tls true) 

docs/contributors/e2e_tests.md

@@ -0,0 +1,143 @@
+# Execute end-to-end tests for local development
+
+To execute the end-to-end (e2e) test suite on your local development system, you have two options:
+
+1. Use a dynamically generated kind server. Each run of the test suite will automatically start a new kind cluster,
+and will be torn down upon completion of tests.
+
+1. Stand up your own minikube or other cluster and use kubeconfig. This option is good for starting a cluster and keeping it
+running even after the test suite has completed.
+
+## Kind without SSL
+
+1. Start a registry server without using SSL
+
+   ```bash
+   ./scripts/start_registry.sh -s kind-registry
+   ```
+
+1. Start the e2e tests:
+
+   ```bash
+   DOCKER_REGISTRY_HOST=localhost:5000 make build e2e SKIPTLS="true" CLUSTER=kind
+   ```
+
+1. Run a specific BDD test using the `TEST` argument to make. Note that this argument uses regular expressions.
+
+   ```bash
+   DOCKER_REGISTRY_HOST=localhost:5000 make build e2e TEST='builds and manipulates bundle and index images' SKIPTLS="true" CLUSTER=kind
+   ```
+
+1. If you want a quick way to ensure that your TEST regex argument will work, you can bypass the 
+make file and use `-dryRun` with `-focus` and see if the regex would trigger your specific test(s).
+
+   ```bash
+   GOFLAGS="-mod=vendor" go run github.com/onsi/ginkgo/ginkgo --v --randomizeAllSpecs --randomizeSuites --race -dryRun -focus 'builds and manipulates bundle and index images' -tags=json1,kind ./test/e2e
+   ```
+
+## Kind with SSL
+
+1. Start a registry server with SSL (NOTE: SSL CA root will be installed and server cert/key will be generated)
+
+   ```bash
+   ./scripts/start_registry.sh kind-registry
+   ```
+
+1. Start the e2e tests:
+
+   ```bash
+   DOCKER_REGISTRY_HOST=localhost:443 make build e2e CLUSTER=kind
+   ```
+
+1. Run a specific BDD test using the `TEST` argument to make. Note that this argument uses regular expressions.
+
+   ```bash
+   DOCKER_REGISTRY_HOST=localhost:443 make build e2e TEST='builds and manipulates bundle and index images' CLUSTER=kind
+   ```
+
+1. If you want a quick way to ensure that your TEST regex argument will work, you can bypass the 
+make file and use `-dryRun` with `-focus` and see if the regex would trigger your specific test(s).
+
+   ```bash
+   GOFLAGS="-mod=vendor" go run github.com/onsi/ginkgo/ginkgo --v --randomizeAllSpecs --randomizeSuites --race -dryRun -focus 'builds and manipulates bundle and index images' -tags=json1,kind ./test/e2e
+   ```
+
+## Minikube (or other type) using kubeconfig without SSL
+
+1. Install `minikube` (see https://minikube.sigs.k8s.io/docs/start/)
+
+1. Create a minikube cluster
+
+   ```bash
+   minikube start
+   ```
+
+1. Start a registry server without using SSL
+
+   ```bash
+   ./scripts/start_registry.sh -s minikube-registry
+   ```
+
+1. Start the e2e tests:
+
+   ```bash
+   KUBECONFIG="$HOME/.kube/config" DOCKER_REGISTRY_HOST=localhost:5000 make build e2e SKIPTLS="true"
+   ```
+
+1. Run a specific BDD test using the `TEST` argument to make. Note that this argument uses regular expressions.
+
+   ```bash
+   KUBECONFIG="$HOME/.kube/config" DOCKER_REGISTRY_HOST=localhost:5000 make build e2e TEST='builds and manipulates bundle and index images' SKIPTLS="true"
+   ```
+
+1. If you want a quick way to ensure that your TEST regex argument will work, you can bypass the 
+make file and use `-dryRun` with `-focus` and see if the regex would trigger your specific test(s).
+
+   ```bash
+   GOFLAGS="-mod=vendor" go run github.com/onsi/ginkgo/ginkgo --v --randomizeAllSpecs --randomizeSuites --race -dryRun -focus 'builds and manipulates bundle and index images' -tags=json1 ./test/e2e
+   ```
+
+TIP: use a non-dynamic `kind` server by using `kind get kubeconfig --name "kind" > /tmp/kindconfig` and set `KUBECONFIG="/tmp/kindconfig"`
+
+## Minikube (or other type) using kubeconfig with SSL
+
+1. Install `minikube` (see https://minikube.sigs.k8s.io/docs/start/)
+
+1. Create a minikube cluster
+
+   ```bash
+   minikube start
+   ```
+
+1. Start a registry server with SSL (NOTE: SSL CA root will be installed and server cert/key will be generated)
+
+   ```bash
+   ./scripts/start_registry.sh minikube-registry
+   ```
+
+1. Start the e2e tests:
+
+   ```bash
+   KUBECONFIG="$HOME/.kube/config" DOCKER_REGISTRY_HOST=localhost:443 make build e2e
+   ```
+
+1. Run a specific BDD test using the `TEST` argument to make. Note that this argument uses regular expressions.
+
+   ```bash
+   KUBECONFIG="$HOME/.kube/config" DOCKER_REGISTRY_HOST=localhost:443 make build e2e TEST='builds and manipulates bundle and index images'
+   ```
+
+1. If you want a quick way to ensure that your TEST regex argument will work, you can bypass the 
+make file and use `-dryRun` with `-focus` and see if the regex would trigger your specific test(s).
+
+   ```bash
+   GOFLAGS="-mod=vendor" go run github.com/onsi/ginkgo/ginkgo --v --randomizeAllSpecs --randomizeSuites --race -dryRun -focus 'builds and manipulates bundle and index images' -tags=json1 ./test/e2e
+   ```
+
+TIP: use a non-dynamic `kind` server by using `kind get kubeconfig --name "kind" > /tmp/kindconfig` and set `KUBECONFIG="/tmp/kindconfig"`
+
+## Known Limitations
+
+Currently the test case `Launch bundle` in test/e2e/bundle_image_test.go assumes that the `opm` executable used in the test is compiled for linux.
+If you run this test on a darwin environment, the kube job will not succeed unless you manually cross compile for linux and include
+the binary at `bin/opm`. 

go.mod

@@ -58,6 +58,8 @@ require (
 	k8s.io/apimachinery v0.20.1
 	k8s.io/client-go v0.20.1
 	k8s.io/kubectl v0.20.0
+	sigs.k8s.io/controller-runtime v0.8.0
+	sigs.k8s.io/kind v0.10.0
 )
 
 replace (