cmd/bundle-extractor/main.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/operator-framework/operator-registry/pkg/sqlite"
+)
+
+func main() {
+	rootCmd := &cobra.Command{
+		Use:   "bundle-extractor",
+		Short: "bundle-extractor",
+		Long:  "Top level CLI for bundle operations",
+	}
+
+	serveCmd := &cobra.Command{
+		Use:   "extract",
+		Short: "Extracts the data in a bundle directory via ConfigMap",
+		Long:  "Extract takes as input a directory containing manifests and writes the per file contents to a ConfipMap",
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runServeCmd,
+	}
+	serveCmd.Flags().Bool("debug", false, "enable debug logging")
+	serveCmd.Flags().StringP("kubeconfig", "k", "", "absolute path to kubeconfig file")
+	serveCmd.Flags().StringP("manifestsdir", "m", "/", "path to directory containing manifests")
+	serveCmd.Flags().StringP("configmapname", "c", "", "name of configmap to write bundle data")
+	serveCmd.Flags().StringP("namespace", "n", "openshift-operator-lifecycle-manager", "namespace to write configmap data")
+	serveCmd.MarkPersistentFlagRequired("configmapname")
+	rootCmd.AddCommand(serveCmd)
+
+	if err := rootCmd.Execute(); err != nil {
+		logrus.Error(err)
+	}
+}
+
+func runServeCmd(cmd *cobra.Command, args []string) error {
+	manifestsDir, err := cmd.Flags().GetString("manifestsdir")
+	if err != nil {
+		return err
+	}
+	kubeconfig, err := cmd.Flags().GetString("kubeconfig")
+	if err != nil {
+		return err
+	}
+	configmapName, err := cmd.Flags().GetString("configmapname")
+	if err != nil {
+		return err
+	}
+	namespace, err := cmd.Flags().GetString("namespace")
+	if err != nil {
+		return err
+	}
+
+	loader := sqlite.NewConfigMapLoaderForDirectory(configmapName, namespace, manifestsDir, kubeconfig)
+	if err := loader.Populate(); err != nil {
+		return fmt.Errorf("error loading manifests from directory: %s", err)
+	}
+
+	return nil
+}

docs/contributors/design-proposals/db-migrations.md

@@ -0,0 +1,61 @@
+# Add database versioning and migrations to output database
+
+Status: Pending
+
+Version: alpha
+
+Implementation owner: TBD
+
+## Abstract
+
+Proposal to add database versioning and migration capability to the output of operator-registry commands
+
+## Motivation
+
+In an attempt at optimizing the registry database usage in OLM to allow the registry database to be stored in a container image, a method is needed to add to the database over time. Since these database files will be long living, it becomes a requirement that the database can migrate to a new schema over time. This proposal attempts to solve that problem by implementing a method with which database versions can be migrated with existing db files over time.
+
+## Proposal
+
+### Migration Method
+
+At a high level, the migration method is relatively straightforward. The database will contain a current migration version and when any additive operation is called with an existing database as input, the first step will be to run the migration to upgrade the database schema to the latest version.
+
+Rather than reinventing the database migration process, operator-registry will use an existing database migration tool that has already defined a set of semantics and conventions. This proposal suggests using the [golang-migrate](https://github.com/golang-migrate/migrate) tool for this purpose. Golang-migrate uses a set of conventions to define migrations.
+
+Firstly, it defines migrations as a set of sql files that live in a flat database folder. Migrations are individually defined as a pair of up and down migration scripts. These up and down scripts each define a method of going to and back from a particular database version and should be opposites semantically. Each script pair has a unique 64 bit unsigned integer identifier followed by an underscore, as well as a title, a direction, and ends in the `.sql` extension: `${version}_${title}.${direction(down/up)}.sql`. The versions are ordered, with the lowest integer value coming first and the latest version defining the current database migration version in the db. For more details on the migration format, please see https://github.com/golang-migrate/migrate/blob/master/MIGRATIONS.md.
+
+```
+ # example migrations folder
+ db_migrations
+ ├── 201909251522_add_users_table.up.sql
+ ├── 201909251522_add_users_table.down.sql
+ ├── 201909251510_first_migration.up.sql
+ └── 201909251510_first_migration.down.sql
+```
+
+Once that migration schema is defined, whenever any additive operation is run against the database we will first use the migrate API to upgrade the schema to the latest version. See https://godoc.org/github.com/golang-migrate/migrate#Migrate.Up for more details on that API.
+
+### Versioning fresh databases
+
+One other consideration of note here is that the operator-registry database is not a long living database in the traditional sense, since the database is often created from scratch. All common migration tools do not generally account for such an edge case, so this proposal also defines a method of initializing the database at a particular migration version. Since the migration version is a matter of convention on file names, we can infer the migration version by parsing the migration folder and finding the latest migration script version. Then, once we initialize the database on startup we can use `golang-migrate` API to force the initial migration version. See https://godoc.org/github.com/golang-migrate/migrate#Migrate.Force for more details on that API.
+
+### Schema Definition
+
+One thing to note is the choice of how the database schema is defined. Currently that schema is defined in source code here in [/pkg/sqlite/load.go](https://github.com/operator-framework/operator-registry/blob/master/pkg/sqlite/load.go#L29) as a list of create table statements.
+
+However, once sql migrations are written the database schema is sometimes generated in two ways (through a migration upgrade or from scratch) as defined above. In that case, there are a few ways to do that. One is to leave the initial schema as is and use the migrations on a clean install as well -- but that means that the schema is not defined in one single human readable place. This can commonly be worked around by using a migration tool that can output an example of the sql schema for the purpose of development.
+
+The other option, and the one that this proposal defines as the solution, is to keep the schema definition in `load.go` in sync with the changes defined in each migration. In this case, it is possible for the database to go out of sync on migration vs from scratch in the case where the migration was written properly. This can be easily mitigated by writing an automated test that always ensures that some initial bundle database can be migrated to a latest version and have the same schema as a clean database.
+
+As a result, this implementation will require writing tests to ensure that the schema versioning is in sync between upgrade and scratch creation.
+
+### Choice of Migration Tool
+
+One point of this proposal is deciding to use the `golang-migrate` project as a method of driving migration conventions and the migrations themselves. Below is a list of criteria that this tool fulfilled when making that choice:
+
+- Popular and well supported
+- Good documentation
+- Supports lots of different database drivers, in the event that we move away from sqlite we can maintain the use of workflows around migrations
+- No need to ship a migration binary, migration can be handled by importing `golang-migrate` and running in source
+- Has a nice API for upgrading and reading the schema version for our scratch scenario that doesn't require us to expect a particular convention of database versioning
+- Allows defining migration versions as timestamps

go.mod

@@ -3,27 +3,29 @@ module github.com/operator-framework/operator-registry
 require (
 	github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6
 	github.com/ghodss/yaml v1.0.0
-	github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect
-	github.com/golang/mock v1.1.1
-	github.com/golang/protobuf v1.2.0
-	github.com/gregjones/httpcache v0.0.0-20190203031600-7a902570cb17 // indirect
+	github.com/golang/mock v1.2.0
+	github.com/golang/protobuf v1.3.1
+	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-health-probe v0.2.1-0.20181220223928-2bf0a5b182db
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/mattn/go-sqlite3 v1.10.0
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190125151539-1e295784b30a
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
 	github.com/otiai10/copy v1.0.1
 	github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect
-	github.com/sirupsen/logrus v1.2.0
-	github.com/spf13/cobra v0.0.3
-	github.com/stretchr/testify v1.2.2
-	golang.org/x/net v0.0.0-20190311183353-d8887717615a
-	golang.org/x/oauth2 v0.0.0-20181105165119-ca4130e427c7
+	github.com/sirupsen/logrus v1.4.2
+	github.com/spf13/cobra v0.0.5
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/grpc v1.23.0
-	gopkg.in/yaml.v2 v2.2.1
-	k8s.io/api v0.0.0-20190118113203-912cbe2bfef3
-	k8s.io/apiextensions-apiserver v0.0.0-20181204003618-e419c5771cdc
-	k8s.io/apimachinery v0.0.0-20190118094746-1525e4dadd2d
-	k8s.io/client-go v8.0.0+incompatible
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.2.2
+	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
+	k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783
+	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
+	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
 )
+
+go 1.13