feat(configmap): add configmap loader

This is to support the older catalog format, which stores data in a
configmap

Author: ecordell

.dockerignore

@@ -0,0 +1,2 @@
+bin
+*.db

.gitignore

@@ -10,3 +10,9 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+# Ignore binaries
+bin
+
+# Ignore sqlite
+*.db

Dockerfile

@@ -1,14 +1,17 @@
-FROM golang:1.10-alpine
+FROM golang:1.10-alpine as builder
 
 RUN apk update && apk add sqlite build-base
 WORKDIR /go/src/github.com/operator-framework/operator-registry
 
 COPY vendor vendor
 COPY cmd cmd
 COPY pkg pkg
-RUN go build --tags json1 -o ./initializer ./cmd/init/...
+COPY Makefile Makefile
+RUN make build
 
 COPY manifests manifests
-RUN ./initializer
+RUN ./bin/initializer -o ./bundles.db
 
+FROM scratch
+COPY --from=builder /go/src/github.com/operator-framework/operator-registry/bundles.db /bundles.db
 

Gopkg.lock

@@ -0,0 +1,15 @@
+.PHONY: build test vendor
+
+all: test build
+
+test:
+	go test --tags json1 -v -race ./pkg/...
+
+build:
+	go build --tags json1 -o ./bin/initializer ./cmd/init/...
+
+image:
+	docker build .
+
+vendor:
+	dep ensure -v

Makefile

@@ -0,0 +1,69 @@
+package registry
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+)
+
+// Scheme is the default instance of runtime.Scheme to which types in the Kubernetes API are already registered.
+var Scheme = runtime.NewScheme()
+
+// Codecs provides access to encoding and decoding for the scheme
+var Codecs = serializer.NewCodecFactory(Scheme)
+
+func DefaultYAMLDecoder() runtime.Decoder {
+	return Codecs.UniversalDeserializer()
+}
+
+func init() {
+	if err := v1alpha1.AddToScheme(Scheme); err != nil {
+		panic(err)
+	}
+
+	if err := v1beta1.AddToScheme(Scheme); err != nil {
+		panic(err)
+	}
+}
+
+func ProvidedAPIs(objs []*unstructured.Unstructured) (map[APIKey]struct{}, error) {
+	provided := map[APIKey]struct{}{}
+	for _, o := range objs {
+		if o.GetObjectKind().GroupVersionKind().Kind == "CustomResourceDefinition" {
+			crd := &apiextensions.CustomResourceDefinition{}
+			if err := runtime.DefaultUnstructuredConverter.FromUnstructured(o.UnstructuredContent(), crd); err != nil {
+				return nil, err
+			}
+			for _, v := range crd.Spec.Versions {
+				provided[APIKey{Group: crd.Spec.Group, Version: v.Name, Kind: crd.Spec.Names.Kind}] = struct{}{}
+			}
+			if crd.Spec.Version != "" {
+				provided[APIKey{Group: crd.Spec.Group, Version: crd.Spec.Version, Kind: crd.Spec.Names.Kind}] = struct{}{}
+			}
+		}
+
+		//TODO: APIServiceDefinitions
+	}
+	return provided, nil
+}
+
+func AllProvidedAPIsInBundle(csv *v1alpha1.ClusterServiceVersion, bundleAPIs map[APIKey]struct{}) error {
+	shouldExist := make(map[APIKey]struct{}, len(csv.Spec.CustomResourceDefinitions.Owned)+len(csv.Spec.APIServiceDefinitions.Owned))
+	for _, crdDef := range csv.Spec.CustomResourceDefinitions.Owned {
+		parts := strings.SplitAfterN(crdDef.Name, ".", 2)
+		shouldExist[APIKey{parts[1], crdDef.Version, crdDef.Kind}] = struct{}{}
+	}
+	//TODO: APIServiceDefinitions
+	for key := range shouldExist {
+		if _, ok := bundleAPIs[key]; !ok {
+			return fmt.Errorf("couldn't find %v in bundle", key)
+		}
+	}
+	return nil
+}

configmap.example.yaml

@@ -1,5 +1,12 @@
 package registry
 
+// APIKey stores GroupVersionKind for use as map keys
+type APIKey struct {
+	Group   string
+	Version string
+	Kind    string
+}
+
 // PackageManifest holds information about a package, which is a reference to one (or more)
 // channels under a single package.
 type PackageManifest struct {

initializer

@@ -0,0 +1,162 @@
+package sqlite
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
+	"github.com/operator-framework/operator-registry/pkg/registry"
+	"github.com/sirupsen/logrus"
+	"k8s.io/api/core/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+const (
+	ConfigMapCRDName     = "customResourceDefinitions"
+	ConfigMapCSVName     = "clusterServiceVersions"
+	ConfigMapPackageName = "packages"
+)
+
+// ConfigMapLoader loads a configmap of resources into the database
+// entries under "customResourceDefinitions" will be parsed as CRDs
+// entries under "clusterServiceVersions"  will be parsed as CSVs
+// entries under "packages" will be parsed as Packages
+type ConfigMapLoader struct {
+	store     registry.Load
+	configMap v1.ConfigMap
+	crds      map[registry.APIKey]*unstructured.Unstructured
+}
+
+var _ SQLPopulator = &ConfigMapLoader{}
+
+func NewSQLLoaderForConfigMap(store registry.Load, configMap v1.ConfigMap) *ConfigMapLoader {
+	return &ConfigMapLoader{
+		store:     store,
+		configMap: configMap,
+		crds: map[registry.APIKey]*unstructured.Unstructured{},
+	}
+}
+
+func (c *ConfigMapLoader) Populate() error {
+	log := logrus.WithFields(logrus.Fields{"configmap": c.configMap.GetName(), "ns": c.configMap.GetNamespace()})
+	log.Info("loading CRDs")
+
+	// first load CRDs into memory; these will be added to the bundle that owns them
+	crdListYaml, ok := c.configMap.Data[ConfigMapCRDName]
+	if !ok {
+		return fmt.Errorf("couldn't find expected key %s in configmap", ConfigMapCRDName)
+	}
+
+	crdListJson, err := yaml.YAMLToJSON([]byte(crdListYaml))
+	if err != nil {
+		log.WithError(err).Debug("error loading CRD list")
+		return err
+	}
+
+	var parsedCRDList []v1beta1.CustomResourceDefinition
+	if err := json.Unmarshal(crdListJson, &parsedCRDList); err!=nil {
+		log.WithError(err).Debug("error parsing CRD list")
+		return err
+	}
+
+	for _, crd := range parsedCRDList {
+		if crd.Spec.Versions == nil && crd.Spec.Version != "" {
+			crd.Spec.Versions = []v1beta1.CustomResourceDefinitionVersion{{Name: crd.Spec.Version, Served: true, Storage: true}}
+		}
+		for _, version := range crd.Spec.Versions {
+			gvk := registry.APIKey{crd.Spec.Group, version.Name, crd.Spec.Names.Kind}
+			log.WithField("gvk", gvk).Debug("loading CRD")
+			if _, ok := c.crds[gvk]; ok {
+				log.WithField("gvk", gvk).Debug("crd added twice")
+				return fmt.Errorf("can't add the same CRD twice in one configmap")
+			}
+			crdUnst, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&crd)
+			if err != nil {
+				log.WithError(err).Debug("error remarshalling crd")
+				return err
+			}
+			c.crds[gvk] = &unstructured.Unstructured{Object: crdUnst}
+		}
+	}
+
+	log.Info("loading Bundles")
+	csvListYaml, ok := c.configMap.Data[ConfigMapCSVName]
+	if !ok {
+		return fmt.Errorf("couldn't find expected key %s in configmap", ConfigMapCSVName)
+	}
+	csvListJson, err := yaml.YAMLToJSON([]byte(csvListYaml))
+	if err != nil {
+		log.WithError(err).Debug("error loading CSV list")
+		return err
+	}
+
+	var parsedCSVList []v1alpha1.ClusterServiceVersion
+	err = json.Unmarshal([]byte(csvListJson), &parsedCSVList)
+	if err != nil {
+		log.WithError(err).Debug("error parsing CSV list")
+		return err
+	}
+
+	for _, csv := range parsedCSVList {
+		log.WithField("csv", csv.Name).Debug("loading CSV")
+		csvUnst, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&csv)
+		if err != nil {
+			log.WithError(err).Debug("error remarshalling csv")
+			return err
+		}
+
+		bundle := []*unstructured.Unstructured{{Object: csvUnst}}
+		for _, owned := range csv.Spec.CustomResourceDefinitions.Owned {
+			split := strings.SplitAfterN(owned.Name, ".", 2)
+			if len(split) < 2 {
+				log.WithError(err).Debug("error parsing owned name")
+				return fmt.Errorf("error parsing owned name")
+			}
+			gvk := registry.APIKey{split[1], owned.Version, owned.Kind}
+			if crdUnst, ok := c.crds[gvk]; !ok {
+				log.WithField("gvk", gvk).WithError(err).Debug("couldn't find owned CRD in crd list")
+			} else {
+				bundle = append(bundle, crdUnst)
+			}
+		}
+
+		if err := c.store.AddOperatorBundle(bundle); err != nil {
+			return err
+		}
+	}
+
+	log.Info("loading Packages")
+	packageListYaml, ok := c.configMap.Data[ConfigMapPackageName]
+	if !ok {
+		return fmt.Errorf("couldn't find expected key %s in configmap", ConfigMapPackageName)
+	}
+
+	packageListJson, err := yaml.YAMLToJSON([]byte(packageListYaml))
+	if err != nil {
+		log.WithError(err).Debug("error loading package list")
+		return err
+	}
+
+	var parsedPackageManifests []registry.PackageManifest
+	err = json.Unmarshal([]byte(packageListJson), &parsedPackageManifests)
+	if err != nil {
+		log.WithError(err).Debug("error parsing package list")
+		return err
+	}
+	for _, packageManifest := range parsedPackageManifests {
+		log.WithField("package", packageManifest.PackageName).Debug("loading package")
+		if err := c.store.AddPackageChannels(packageManifest); err != nil {
+			return err
+		}
+	}
+
+	log.Info("extracting provided API information")
+	if err := c.store.AddProvidedApis(); err != nil {
+		return err
+	}
+	return nil
+}