Skip to content

Commit 645f15f

Browse files
anik120anik120
committed
stand up registry for e2e
Signed-off-by: Anik Bhattacharjee <[email protected]>
1 parent 2926297 commit 645f15f

File tree

8 files changed

+216
-16
lines changed

8 files changed

+216
-16
lines changed

‎Makefile

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,11 +118,14 @@ UNIT_TEST_DIRS=$(shell go list ./... | grep -v /test/)
118118
test-unit: $(SETUP_ENVTEST) #HELP Run the unit tests
119119
eval $$($(SETUP_ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out
120120

121+
image-registry: ## Setup in-cluster image registry
122+
./test/tools/imageregistry/registry.sh
123+
121124
.PHONY: test-e2e
122125
test-e2e: KIND_CLUSTER_NAME=operator-controller-e2e
123126
test-e2e: KUSTOMIZE_BUILD_DIR=config/e2e
124127
test-e2e: GO_BUILD_FLAGS=-cover
125-
test-e2e: run kind-load-test-artifacts e2e e2e-coverage undeploy kind-clean #HELP Run e2e test suite on local kind cluster
128+
test-e2e: run image-registry kind-load-test-artifacts e2e e2e-coverage undeploy kind-clean #HELP Run e2e test suite on local kind cluster
126129

127130
.PHONY: operator-developer-e2e
128131
operator-developer-e2e: KIND_CLUSTER_NAME=operator-controller-op-dev-e2e #EXHELP Run operator-developer e2e on local kind cluster
@@ -161,7 +164,6 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images
161164
$(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0
162165
$(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0
163166
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0
164-
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog:e2e
165167
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.37.0 --name $(KIND_CLUSTER_NAME)
166168
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0 --name $(KIND_CLUSTER_NAME)
167169
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v0.65.1 --name $(KIND_CLUSTER_NAME)
@@ -170,7 +172,6 @@ kind-load-test-artifacts: $(KIND) #EXHELP Load the e2e testdata container images
170172
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v1.2.0 --name $(KIND_CLUSTER_NAME)
171173
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator:v2.0.0 --name $(KIND_CLUSTER_NAME)
172174
$(KIND) load docker-image localhost/testdata/bundles/plain-v0/plain:v0.1.0 --name $(KIND_CLUSTER_NAME)
173-
$(KIND) load docker-image localhost/testdata/catalogs/test-catalog:e2e --name $(KIND_CLUSTER_NAME)
174175

175176
.PHONY: deploy-local-registry
176177
deploy-local-registry: #EXHELP Deploy local registry.

‎test/e2e/e2e_suite_test.go

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
"k8s.io/utils/env"
1414

1515
"k8s.io/apimachinery/pkg/api/errors"
16-
apimeta "k8s.io/apimachinery/pkg/api/meta"
1716
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1817
"k8s.io/apimachinery/pkg/runtime"
1918
"k8s.io/apimachinery/pkg/types"
@@ -35,11 +34,11 @@ var (
3534

3635
const (
3736
testCatalogRefEnvVar = "TEST_CATALOG_IMAGE"
38-
testCatalogRefDefault = "localhost/testdata/catalogs/test-catalog:e2e"
37+
testCatalogRefDefault = "docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e"
3938
testCatalogName = "test-catalog"
4039
)
4140

42-
// returns the image reference for the test, checking for environment variable substitution, with a default of localhost/testdata/catalogs/test-catalog:e2e
41+
// returns the image reference for the test, checking for environment variable substitution, with a default of docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e
4342
func getCatalogImageRef() string {
4443
if s := os.Getenv(testCatalogRefEnvVar); s != "" {
4544
return s
@@ -78,16 +77,6 @@ var _ = BeforeSuite(func() {
7877
operatorCatalog, err = createTestCatalog(ctx, testCatalogName, getCatalogImageRef())
7978
Expect(err).ToNot(HaveOccurred())
8079

81-
// TODO: REMOVE THIS. This should not be necessary if we have idiomatic APIs. Kubernetes is supposed
82-
// to be eventually consistent. Waits for preconditions like this hide bugs.
83-
Eventually(func(g Gomega) {
84-
err := c.Get(ctx, types.NamespacedName{Name: operatorCatalog.Name}, operatorCatalog)
85-
g.Expect(err).ToNot(HaveOccurred())
86-
cond := apimeta.FindStatusCondition(operatorCatalog.Status.Conditions, catalogd.TypeUnpacked)
87-
g.Expect(cond).ToNot(BeNil())
88-
g.Expect(cond.Status).To(Equal(metav1.ConditionTrue))
89-
g.Expect(cond.Reason).To(Equal(catalogd.ReasonUnpackSuccessful))
90-
}).Should(Succeed())
9180
})
9281

9382
var _ = AfterSuite(func() {

‎test/tools/imageregistry/catalogd_controller_patch.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
spec:
2+
template:
3+
spec:
4+
containers:
5+
- name: manager
6+
volumeMounts:
7+
- mountPath: /etc/ssl/certs/
8+
name: ca-certs
9+
readOnly: true
10+
volumes:
11+
- name: ca-certs
12+
configMap:
13+
name: docker-registry.operator-controller-e2e.svc
14+
defaultMode: 0644
15+
optional: false
16+
items:
17+
- key: ca-certificates.crt
18+
path: ca-certificates.crt

‎test/tools/imageregistry/imagebuilder.yaml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: kaniko
5+
namespace: operator-controller-e2e
6+
spec:
7+
template:
8+
spec:
9+
containers:
10+
- name: kaniko
11+
image: gcr.io/kaniko-project/executor:latest
12+
args: ["--dockerfile=/workspace/test-catalog.Dockerfile",
13+
"--context=/workspace/",
14+
"--destination=docker-registry.operator-controller-e2e.svc:5000/test-catalog:e2e"]
15+
volumeMounts:
16+
- name: certs
17+
mountPath: /etc/ssl/certs/
18+
readOnly: true
19+
- name: dockerfile
20+
mountPath: /workspace/
21+
- name: build-contents
22+
mountPath: /workspace/test-catalog/
23+
restartPolicy: Never
24+
volumes:
25+
- name: dockerfile
26+
configMap:
27+
name: operator-controller-e2e.dockerfile
28+
items:
29+
- key: test-catalog.Dockerfile
30+
path: test-catalog.Dockerfile
31+
- name: build-contents
32+
configMap:
33+
name: operator-controller-e2e.build-contents
34+
- name: certs
35+
configMap:
36+
name: regcerts
37+
defaultMode: 0644
38+
optional: false
39+
items:
40+
- key: ca-certificates.crt
41+
path: ca-certificates.crt

‎test/tools/imageregistry/imgreg.yaml

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: operator-controller-e2e
5+
---
6+
apiVersion: cert-manager.io/v1
7+
kind: Issuer
8+
metadata:
9+
name: selfsigned-issuer
10+
namespace: operator-controller-e2e
11+
spec:
12+
selfSigned: {}
13+
---
14+
apiVersion: cert-manager.io/v1
15+
kind: Certificate
16+
metadata:
17+
name: operator-controller-e2e-registry
18+
namespace: operator-controller-e2e
19+
spec:
20+
secretName: operator-controller-e2e-registry
21+
isCA: true
22+
dnsNames:
23+
- docker-registry.operator-controller-e2e.svc
24+
privateKey:
25+
algorithm: ECDSA
26+
size: 256
27+
issuerRef:
28+
name: selfsigned-issuer
29+
kind: Issuer
30+
group: cert-manager.io
31+
---
32+
apiVersion: apps/v1
33+
kind: Deployment
34+
metadata:
35+
name: docker-registry
36+
namespace: operator-controller-e2e
37+
labels:
38+
app: registry
39+
spec:
40+
replicas: 1
41+
selector:
42+
matchLabels:
43+
app: registry
44+
template:
45+
metadata:
46+
labels:
47+
app: registry
48+
spec:
49+
containers:
50+
- name: registry
51+
image: registry:2
52+
volumeMounts:
53+
- name: certs-vol
54+
mountPath: "/certs"
55+
env:
56+
- name: REGISTRY_HTTP_TLS_CERTIFICATE
57+
value: "/certs/tls.crt"
58+
- name: REGISTRY_HTTP_TLS_KEY
59+
value: "/certs/tls.key"
60+
volumes:
61+
- name: certs-vol
62+
secret:
63+
secretName: operator-controller-e2e-registry
64+
---
65+
apiVersion: v1
66+
kind: Service
67+
metadata:
68+
name: docker-registry
69+
namespace: operator-controller-e2e
70+
spec:
71+
selector:
72+
app: registry
73+
ports:
74+
- port: 5000
75+
targetPort: 5000

‎test/tools/imageregistry/registry.sh

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
# registry.sh will create an in-cluster image registry useful for end-to-end testing
6+
# of catalogd's unpacking process. It does a few things:
7+
# 1. Installs cert-manager for creating a self-signed certificate for the image registry
8+
# 2. Creates all the resources necessary for deploying the image registry in the operator-controller-e2e namespace
9+
# 3. Creates a ConfigMap containing the CA cert for the image registry to be used by the catalogd-controller-manager
10+
# 4. Creates a ConfigMap containing the CA cert for the image registry to be used by the kaniko pod
11+
# 5. Creates ConfigMaps containing the test catalog + Dockerfile to be mounted to the kaniko pod
12+
# 6. Waits for kaniko pod to have Phase == Succeeded, indicating the test catalog image has been built + pushed
13+
# to the test image registry
14+
# Usage:
15+
# registry.sh
16+
17+
# Install cert-manager
18+
kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/v1.13.1/cert-manager.yaml"
19+
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s
20+
21+
# create the image registry with all the certs
22+
kubectl apply -f test/tools/imageregistry/imgreg.yaml
23+
kubectl wait -n operator-controller-e2e --for=condition=Available deployment/docker-registry --timeout=60s
24+
25+
# get cert value
26+
certval=$(kubectl -n operator-controller-e2e get secret/operator-controller-e2e-registry -o=jsonpath='{.data.ca\.crt}' | base64 -d | sed 's/^/ /')
27+
28+
kubectl apply -f - << EOF
29+
apiVersion: v1
30+
kind: Namespace
31+
metadata:
32+
name: catalogd-system
33+
EOF
34+
# create a ConfigMap that contains the CA certs for the image registry
35+
# This one is created for the catalogd-controller-manager
36+
kubectl apply -f - << EOF
37+
apiVersion: v1
38+
kind: ConfigMap
39+
metadata:
40+
namespace: catalogd-system
41+
name: docker-registry.operator-controller-e2e.svc
42+
data:
43+
"ca-certificates.crt": |
44+
${certval}
45+
EOF
46+
47+
# create a ConfigMap that contains the CA certs for the image registry
48+
# this one is created for the kaniko pod
49+
kubectl apply -f - << EOF
50+
apiVersion: v1
51+
kind: ConfigMap
52+
metadata:
53+
namespace: operator-controller-e2e
54+
name: regcerts
55+
data:
56+
"ca-certificates.crt": |
57+
${certval}
58+
EOF
59+
60+
# Load the testdata onto the cluster as a configmap so it can be used with kaniko
61+
kubectl create configmap -n operator-controller-e2e --from-file=testdata/catalogs/test-catalog.Dockerfile operator-controller-e2e.dockerfile
62+
kubectl create configmap -n operator-controller-e2e --from-file=testdata/catalogs/test-catalog operator-controller-e2e.build-contents
63+
64+
# Create the kaniko pod to build the test image and push it to the test registry.
65+
kubectl apply -f test/tools/imageregistry/imagebuilder.yaml
66+
kubectl wait --for=condition=Complete -n operator-controller-e2e jobs/kaniko --timeout=60s
67+
68+
kubectl patch deployment catalogd-controller-manager -n catalogd-system --patch-file test/tools/imageregistry/catalogd_controller_patch.yaml
69+
kubectl delete pod --force -n catalogd-system $(kubectl get pods --no-headers -o custom-columns=":metadata.name" -n catalogd-system | grep catalogd-controller-manager)
70+
kubectl rollout status deployment/catalogd-controller-manager -n catalogd-system --timeout=540s

‎testdata/catalogs/test-catalog.Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
11
FROM scratch
22
ADD test-catalog /configs
3+
4+
# Set DC-specific label for the location of the DC root directory
5+
# in the image
6+
LABEL operators.operatorframework.io.index.configs.v1=/configs

‎testdata/catalogs/test-catalog/.indexignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/expected_all.json
2+
..*

0 commit comments

Comments
 (0)