Bundle migrations in binary

Use the go-bindata tool to generate a golang migrations file to
compile the content of the db_migrations folder into the bundle. This
ensures that the compiled binaries are portable and do not require
around the migrations themselves along with the binary.

Also updated the makefile to pull the go-bindata file and generate the
`migration.go` file as part of `make build`

Author: kevinrizza

Makefile

@@ -3,7 +3,7 @@ CMDS  := $(addprefix bin/, $(shell go list $(MOD_FLAGS) ./cmd/... | xargs -I{} b
 
 .PHONY: build test vendor clean
 
-all: clean test build
+all: clean install-go-bindata test build
 
 $(CMDS):
 	go build $(MOD_FLAGS) $(extra_flags) -o $@ ./cmd/$(shell basename $@)
@@ -25,6 +25,12 @@ image-upstream:
 vendor:
 	go mod vendor
 
+install-go-bindata:
+	go get -u github.com/go-bindata/go-bindata/...
+
+generate-migration-bundle:
+	go-bindata -pkg sqlite -o ./pkg/sqlite/migrations.go ./pkg/sqlite/db_migrations/
+
 codegen:
 	protoc -I pkg/api/ --go_out=plugins=grpc:pkg/api pkg/api/*.proto
 	protoc -I pkg/api/grpc_health_v1 --go_out=plugins=grpc:pkg/api/grpc_health_v1 pkg/api/grpc_health_v1/*.proto

cmd/configmap-server/main.go

@@ -98,7 +98,7 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 		logger.Fatalf("error getting configmap: %s", err)
 	}
 
-	sqlLoader, err := sqlite.NewSQLLiteLoader(dbName, "")
+	sqlLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
 	if err != nil {
 		return err
 	}

cmd/initializer/main.go

@@ -53,7 +53,7 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	dbLoader, err := sqlite.NewSQLLiteLoader(outFilename, "")
+	dbLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(outFilename))
 	if err != nil {
 		return err
 	}

pkg/appregistry/dbloader.go

@@ -11,7 +11,7 @@ import (
 )
 
 func NewDbLoader(dbName string, logger *logrus.Entry) (*dbLoader, error) {
-	sqlLoader, err := sqlite.NewSQLLiteLoader(dbName, "")
+	sqlLoader, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
 	if err != nil {
 		return nil, err
 	}

pkg/server/server_test.go

@@ -29,7 +29,7 @@ func server() {
 	}
 	s := grpc.NewServer()
 
-	load, err := sqlite.NewSQLLiteLoader(dbName, "../sqlite/db_migrations")
+	load, err := sqlite.NewSQLLiteLoader(sqlite.WithDBName(dbName))
 	if err != nil {
 		logrus.Fatal(err)
 	}

pkg/sqlite/configmap_test.go

@@ -16,7 +16,7 @@ import (
 func TestConfigMapLoader(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
 
-	store, err := NewSQLLiteLoader("test.db", "./db_migrations")
+	store, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer os.Remove("test.db")
 
@@ -34,7 +34,7 @@ func TestConfigMapLoader(t *testing.T) {
 }
 
 func TestQuerierForConfigmap(t *testing.T) {
-	load, err := NewSQLLiteLoader("test.db", "./db_migrations")
+	load, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer os.Remove("test.db")
 

pkg/sqlite/db_options.go

@@ -0,0 +1,23 @@
+package sqlite
+
+type DbOptions struct {
+	// OutFileName is used to define the database file name that is generated from the loader
+	OutFileName string
+
+	// Migrator refers to the SQL migrator used to initialize the database with
+	MigrationsPath string
+}
+
+type DbOption func(*DbOptions)
+
+func WithDBName(name string) DbOption {
+	return func(o *DbOptions) {
+        o.OutFileName = name
+    }
+}
+
+func WithMigrationsPath(path string) DbOption {
+	return func(o *DbOptions) {
+        o.MigrationsPath = path
+    }
+}

pkg/sqlite/directory_test.go

@@ -18,7 +18,7 @@ import (
 func TestDirectoryLoader(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
 
-	store, err := NewSQLLiteLoader("test.db", "./db_migrations")
+	store, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer func() {
 		if err := os.Remove("test.db"); err != nil {
@@ -33,7 +33,7 @@ func TestDirectoryLoader(t *testing.T) {
 func TestDirectoryLoaderWithBadManifests(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
 
-	store, err := NewSQLLiteLoader("test.db", "./db_migrations")
+	store, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer func() {
 		if err := os.Remove("test.db"); err != nil {
@@ -72,7 +72,7 @@ func TestDirectoryLoaderWithBadManifests(t *testing.T) {
 }
 
 func TestQuerierForDirectory(t *testing.T) {
-	load, err := NewSQLLiteLoader("test.db", "./db_migrations")
+	load, err := NewSQLLiteLoader(WithDBName("test.db"))
 	require.NoError(t, err)
 	defer func() {
 		if err := os.Remove("test.db"); err != nil {

pkg/sqlite/load.go

@@ -20,8 +20,13 @@ type SQLLoader struct {
 
 var _ registry.Load = &SQLLoader{}
 
-func NewSQLLiteLoader(outFilename, migrationsPath string) (*SQLLoader, error) {
-	db, err := sql.Open("sqlite3", outFilename) // TODO: ?immutable=true
+func NewSQLLiteLoader(opts ...DbOption) (*SQLLoader, error) {
+	options := DbOptions{}
+    for _, o := range opts {
+        o(&options)
+    }
+
+	db, err := sql.Open("sqlite3", options.OutFileName) // TODO: ?immutable=true
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +84,12 @@ func NewSQLLiteLoader(outFilename, migrationsPath string) (*SQLLoader, error) {
 	}
 
 	// Apply the current latest database version to keep net new databases in sync with upgradeable ones
-	migrator := NewSQLLiteMigrator(db, migrationsPath)
+	migrator, err := NewSQLLiteMigrator(db, options.MigrationsPath)
+	if err != nil {
+		return nil, err
+	}
+	defer migrator.CleanUpMigrator()
+
 	err = migrator.InitMigrationVersion()
 	if err != nil {
 		return nil, err

pkg/sqlite/load_test.go

@@ -131,7 +131,7 @@ func TestAddPackageChannels(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.description, func(t *testing.T) {
 			db := fmt.Sprintf("%d.db", rand.Int())
-			store, err := NewSQLLiteLoader(db, "./db_migrations")
+			store, err := NewSQLLiteLoader(WithDBName(db))
 			require.NoError(t, err)
 			defer func() {
 				if err := os.Remove(db); err != nil {

pkg/sqlite/migrations.go

@@ -3,6 +3,7 @@ package sqlite
 import (
 	"database/sql"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -14,26 +15,70 @@ import (
 	_ "github.com/golang-migrate/migrate/v4/source/file" // indirect import required by golang-migrate package
 )
 
-const (
-	// Hardcoded path where the db_migrations live
-	defaultMigrationsPath = "./pkg/sqlite/db_migrations"
-)
-
 type SQLMigrator struct {
 	db             *sql.DB
 	migrationsPath string
+	generated bool
 }
 
 // NewSQLLiteMigrator returns a SQLMigrator. The SQLMigrator takes a sql database and directory for migrations
 // and exposes a set of functions that allow the golang-migrate project to apply migrations to that database.
-func NewSQLLiteMigrator(db *sql.DB, migrationsPath string) *SQLMigrator {
+func NewSQLLiteMigrator(db *sql.DB, migrationsPath string) (*SQLMigrator, error) {
+	// If no migrations folder is set, use the generated migrations
 	if migrationsPath == "" {
-		migrationsPath = defaultMigrationsPath
+		// Create a temp dir for the generated migrations
+		tempDir, err := ioutil.TempDir(".", "db_migrations_")
+		if err != nil {
+			return nil, err
+		}
+
+		migrationsFolder := "pkg/sqlite/db_migrations"
+
+		dirData, err := AssetDir(migrationsFolder)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, file := range dirData {
+			fileData, err := Asset(fmt.Sprintf("%s/%s", migrationsFolder, file))
+			if err != nil {
+				return nil, err
+			}
+			
+			f, err := os.Create(fmt.Sprintf("%s/%s", tempDir, file))
+			if err != nil {
+				return nil, err
+			}
+			defer f.Close()
+
+			_, err = f.Write(fileData)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return &SQLMigrator{
+			db:             db,
+			migrationsPath: tempDir,
+			generated: true,
+		}, nil
 	}
+
 	return &SQLMigrator{
 		db:             db,
 		migrationsPath: migrationsPath,
+		generated: false,
+	}, nil
+}
+
+// CleanUpMigrator deletes any unnecessary data generated just for the scope of the migrator. 
+// Call this function once the scope of the Migrator is no longer required
+func (m *SQLMigrator) CleanUpMigrator() {
+	if m.generated {
+		os.RemoveAll(m.migrationsPath)
 	}
+
+	return
 }
 
 // InitMigrationVersion parses the db_migrations for the latest migration version, then applies that

pkg/sqlite/migrator.go

@@ -11,11 +11,12 @@ import (
 func TestValidMigratorVersion(t *testing.T) {
 	migrationsPath := "./db_migrations"
 
-	store, err := NewSQLLiteLoader("test.db", migrationsPath)
+	store, err := NewSQLLiteLoader(WithDBName("test.db"), WithMigrationsPath(migrationsPath))
 	require.NoError(t, err)
 	defer os.Remove("test.db")
 
-	migrator := NewSQLLiteMigrator(store.db, migrationsPath)
+	migrator, err := NewSQLLiteMigrator(store.db, migrationsPath)
+	require.NoError(t, err, "Unable to initialize migrator")
 
 	version, err := migrator.CurrentVersion()
 	require.NoError(t, err, "Could not parse latest migration version from db_migrations folder")
@@ -27,11 +28,12 @@ func TestGetMigrationVersion(t *testing.T) {
 	expectedVersion := uint(200412250000)
 	migrationsPath := "./testdata/test_db_migrations/valid"
 
-	store, err := NewSQLLiteLoader("test.db", migrationsPath)
+	store, err := NewSQLLiteLoader(WithDBName("test.db"), WithMigrationsPath(migrationsPath))
 	require.NoError(t, err)
 	defer os.Remove("test.db")
 
-	migrator := NewSQLLiteMigrator(store.db, migrationsPath)
+	migrator, err := NewSQLLiteMigrator(store.db, migrationsPath)
+	require.NoError(t, err, "Unable to initialize migrator")
 
 	version, err := migrator.CurrentVersion()
 	require.NoError(t, err, "Could not parse latest migration version from db_migrations folder")
@@ -43,7 +45,18 @@ func TestGetMigrationVersion(t *testing.T) {
 func TestGetMigrationError(t *testing.T) {
 	migrationsPath := "./testdata/test_db_migrations/invalid"
 
-	_, err := NewSQLLiteLoader("test.db", migrationsPath)
+	_, err := NewSQLLiteLoader(WithDBName("test.db"), WithMigrationsPath(migrationsPath))
 	require.Error(t, err)
 	defer os.Remove("test.db")
 }
+
+func TestGeneratedMigrations(t *testing.T) {
+	store, err := NewSQLLiteLoader(WithDBName("test.db"))
+	require.NoError(t, err)
+	defer os.Remove("test.db")
+	
+	migrator, err := NewSQLLiteMigrator(store.db, "")
+	defer migrator.CleanUpMigrator()
+
+	require.NoError(t, err, "Unable to initialize migrator with generated migrations")
+}