feature: opm (index|registry) prune command

This makes it possible for a user to prune an index image or an operator
database referencing a large collection of operator bundles.

Author: djzager

cmd/opm/index/cmd.go

@@ -24,4 +24,5 @@ func AddCommand(parent *cobra.Command) {
 	cmd.AddCommand(newIndexDeleteCmd())
 	addIndexAddCmd(cmd)
 	cmd.AddCommand(newIndexExportCmd())
+	cmd.AddCommand(newIndexPruneCmd())
 }

cmd/opm/index/prune.go

@@ -0,0 +1,120 @@
+package index
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/operator-framework/operator-registry/pkg/containertools"
+	"github.com/operator-framework/operator-registry/pkg/lib/indexer"
+)
+
+func newIndexPruneCmd() *cobra.Command {
+	indexCmd := &cobra.Command{
+		Use:   "prune",
+		Short: "prune an index of all but specified packages",
+		Long:  `prune an index of all but specified packages`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runIndexPruneCmdFunc,
+	}
+
+	indexCmd.Flags().Bool("debug", false, "enable debug logging")
+	indexCmd.Flags().Bool("generate", false, "if enabled, just creates the dockerfile and saves it to local disk")
+	indexCmd.Flags().StringP("out-dockerfile", "d", "", "if generating the dockerfile, this flag is used to (optionally) specify a dockerfile name")
+	indexCmd.Flags().StringP("from-index", "f", "", "index to prune")
+	if err := indexCmd.MarkFlagRequired("from-index"); err != nil {
+		logrus.Panic("Failed to set required `from-index` flag for `index prune`")
+	}
+	indexCmd.Flags().StringSliceP("packages", "p", nil, "comma separated list of packages to keep")
+	if err := indexCmd.MarkFlagRequired("packages"); err != nil {
+		logrus.Panic("Failed to set required `packages` flag for `index prune`")
+	}
+	indexCmd.Flags().StringP("binary-image", "i", "", "container image for on-image `opm` command")
+	indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
+	indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
+	indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
+
+	if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
+		logrus.Panic(err.Error())
+	}
+
+	return indexCmd
+
+}
+
+func runIndexPruneCmdFunc(cmd *cobra.Command, args []string) error {
+	generate, err := cmd.Flags().GetBool("generate")
+	if err != nil {
+		return err
+	}
+
+	outDockerfile, err := cmd.Flags().GetString("out-dockerfile")
+	if err != nil {
+		return err
+	}
+
+	fromIndex, err := cmd.Flags().GetString("from-index")
+	if err != nil {
+		return err
+	}
+
+	packages, err := cmd.Flags().GetStringSlice("packages")
+	if err != nil {
+		return err
+	}
+
+	binaryImage, err := cmd.Flags().GetString("binary-image")
+	if err != nil {
+		return err
+	}
+
+	containerTool, err := cmd.Flags().GetString("container-tool")
+	if err != nil {
+		return err
+	}
+
+	if containerTool == "none" {
+		return fmt.Errorf("none is not a valid container-tool for index prune")
+	}
+
+	tag, err := cmd.Flags().GetString("tag")
+	if err != nil {
+		return err
+	}
+
+	permissive, err := cmd.Flags().GetBool("permissive")
+	if err != nil {
+		return err
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"packages": packages})
+
+	logger.Info("pruning the index")
+
+	indexPruner := indexer.NewIndexPruner(containertools.NewContainerTool(containerTool, containertools.PodmanTool), logger)
+
+	request := indexer.PruneFromIndexRequest{
+		Generate:          generate,
+		FromIndex:         fromIndex,
+		BinarySourceImage: binaryImage,
+		OutDockerfile:     outDockerfile,
+		Packages:          packages,
+		Tag:               tag,
+		Permissive:        permissive,
+	}
+
+	err = indexPruner.PruneFromIndex(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/opm/registry/cmd.go

@@ -23,6 +23,7 @@ func NewOpmRegistryCmd() *cobra.Command {
 	rootCmd.AddCommand(newRegistryServeCmd())
 	rootCmd.AddCommand(newRegistryAddCmd())
 	rootCmd.AddCommand(newRegistryRmCmd())
+	rootCmd.AddCommand(newRegistryPruneCmd())
 
 	return rootCmd
 }

cmd/opm/registry/prune.go

@@ -0,0 +1,69 @@
+package registry
+
+import (
+	"github.com/operator-framework/operator-registry/pkg/lib/registry"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func newRegistryPruneCmd() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:   "prune",
+		Short: "prune an operator registry DB of all but specified packages",
+		Long:  `prune an operator registry DB of all but specified packages`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runRegistryPruneCmdFunc,
+	}
+
+	rootCmd.Flags().Bool("debug", false, "enable debug logging")
+	rootCmd.Flags().StringP("database", "d", "bundles.db", "relative path to database file")
+	rootCmd.Flags().StringSliceP("packages", "p", []string{}, "comma separated list of package names to be kept")
+	if err := rootCmd.MarkFlagRequired("packages"); err != nil {
+		logrus.Panic("Failed to set required `packages` flag for `registry rm`")
+	}
+	rootCmd.Flags().Bool("permissive", false, "allow registry load errors")
+
+	return rootCmd
+}
+
+func runRegistryPruneCmdFunc(cmd *cobra.Command, args []string) error {
+	fromFilename, err := cmd.Flags().GetString("database")
+	if err != nil {
+		return err
+	}
+	packages, err := cmd.Flags().GetStringSlice("packages")
+	if err != nil {
+		return err
+	}
+	permissive, err := cmd.Flags().GetBool("permissive")
+	if err != nil {
+		return err
+	}
+
+	request := registry.PruneFromRegistryRequest{
+		Packages:      packages,
+		InputDatabase: fromFilename,
+		Permissive:    permissive,
+	}
+
+	logger := logrus.WithFields(logrus.Fields{"packages": packages})
+
+	logger.Info("pruning from the registry")
+
+	registryPruner := registry.NewRegistryPruner(logger)
+
+	err = registryPruner.PruneFromRegistry(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

docs/design/opm-tooling.md

@@ -34,6 +34,14 @@ Great! The existing `test-registry.db` file is updated. Now we have a registry t
 
 Calling this on our existing test registry removes all versions of the prometheus operator entirely from the database.
 
+#### prune
+
+`opm` supports specifying which packages should be kept in an operator database. For example:
+
+`opm registry prune -p "prometheus" -d "test-registry.db"`
+
+Would remove all but the `prometheus` package from the operator database.
+
 #### serve
 
 `opm` also includes a command to connect to an existing database and serve a `gRPC` API that handles requests for data about the registry:
@@ -93,6 +101,14 @@ Like `opm registry rm`, this command will remove all versions an entire operator
 
 This will result in the tagged container image `quay.io/operator-framework/monitoring-index:1.0.2` with a registry that no longer contains the `prometheus` operator at all.
 
+#### prune
+
+`opm index prune` allows the user to specify which operator packages should be maintained in an index. For example:
+
+`opm index prune -p "prometheus" --from-index quay.io/operator-framework/example-index:1.0.0 --tag quay.io/operator-framework/example-index:1.0.1`
+
+Would remove all but the `prometheus` package from the index.
+
 #### export
 
 `opm index export` will export a package from an index image into a directory. The format of this directory will match the appregistry manifest format: containing all versions of the package in the index along with a `package.yaml` file. This command takes an `--index` flag that points to an index image, a `--package` flag that states a package name, an optional `--download-folder` as the export location (default is `./downloaded`), and just as the other index commands it takes a `--container-tool` flag. Ex:

pkg/lib/indexer/indexer.go

@@ -41,6 +41,7 @@ type ImageIndexer struct {
 	ImageReader         containertools.ImageReader
 	RegistryAdder       registry.RegistryAdder
 	RegistryDeleter     registry.RegistryDeleter
+	RegistryPruner      registry.RegistryPruner
 	ContainerTool       containertools.ContainerTool
 	Logger              *logrus.Entry
 }
@@ -205,6 +206,82 @@ func (i ImageIndexer) DeleteFromIndex(request DeleteFromIndexRequest) error {
 	return nil
 }
 
+// PruneFromIndexRequest defines the parameters to send to the PruneFromIndex API
+type PruneFromIndexRequest struct {
+	Generate          bool
+	Permissive        bool
+	BinarySourceImage string
+	FromIndex         string
+	OutDockerfile     string
+	Tag               string
+	Packages          []string
+}
+
+func (i ImageIndexer) PruneFromIndex(request PruneFromIndexRequest) error {
+	buildDir, outDockerfile, cleanup, err := buildContext(request.Generate, request.OutDockerfile)
+	defer cleanup()
+	if err != nil {
+		return err
+	}
+
+	// set a temp directory for unpacking an image
+	// this is in its own function context so that the deferred cleanup runs before we do a docker build
+	// which prevents the full contents of the previous image from being in the build context
+	var databasePath string
+	if err := func () error {
+		tmpDir, err := ioutil.TempDir("./", tmpDirPrefix)
+		if err != nil {
+
+			return err
+		}
+		defer os.RemoveAll(tmpDir)
+
+		databaseFile, err := i.getDatabaseFile(tmpDir, request.FromIndex)
+		if err != nil {
+			return err
+		}
+		// copy the index to the database folder in the build directory
+		if databasePath, err = copyDatabaseTo(databaseFile, filepath.Join(buildDir, defaultDatabaseFolder)); err != nil {
+			return err
+		}
+		return nil
+	}(); err != nil {
+		return err
+	}
+
+	// Run opm registry prune on the database
+	pruneFromRegistryReq := registry.PruneFromRegistryRequest{
+		Packages:      request.Packages,
+		InputDatabase: databasePath,
+		Permissive:    request.Permissive,
+	}
+
+	// Prune the bundles from the registry
+	err = i.RegistryPruner.PruneFromRegistry(pruneFromRegistryReq)
+	if err != nil {
+		return err
+	}
+
+	// generate the dockerfile
+	dockerfile := i.DockerfileGenerator.GenerateIndexDockerfile(request.BinarySourceImage, databasePath)
+	err = write(dockerfile, outDockerfile, i.Logger)
+	if err != nil {
+		return err
+	}
+
+	if request.Generate {
+		return nil
+	}
+
+	// build the dockerfile
+	err = build(outDockerfile, request.Tag, i.CommandRunner, i.Logger)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (i ImageIndexer) getDatabaseFile(workingDir, fromIndex string) (string, error) {
 	if fromIndex == "" {
 		return path.Join(workingDir, defaultDatabaseFile), nil

pkg/lib/indexer/interfaces.go

@@ -63,3 +63,20 @@ func NewIndexExporter(containerTool containertools.ContainerTool, logger *logrus
 		Logger:              logger,
 	}
 }
+
+// IndexPruner prunes operators out of an index
+type IndexPruner interface {
+	PruneFromIndex(PruneFromIndexRequest) error
+}
+
+func NewIndexPruner(containerTool containertools.ContainerTool, logger *logrus.Entry) IndexPruner {
+	return ImageIndexer{
+		DockerfileGenerator: containertools.NewDockerfileGenerator(logger),
+		CommandRunner:       containertools.NewCommandRunner(containerTool, logger),
+		LabelReader:         containertools.NewLabelReader(containerTool, logger),
+		RegistryPruner:      registry.NewRegistryPruner(logger),
+		ImageReader:         containertools.NewImageReader(containerTool, logger),
+		ContainerTool:       containerTool,
+		Logger:              logger,
+	}
+}

pkg/lib/registry/interfaces.go

@@ -26,3 +26,13 @@ func NewRegistryDeleter(logger *logrus.Entry) RegistryDeleter {
 		Logger: logger,
 	}
 }
+
+type RegistryPruner interface {
+	PruneFromRegistry(PruneFromRegistryRequest) error
+}
+
+func NewRegistryPruner(logger *logrus.Entry) RegistryPruner {
+	return RegistryUpdater{
+		Logger: logger,
+	}
+}

pkg/lib/registry/registry.go

@@ -177,3 +177,55 @@ func (r RegistryUpdater) DeleteFromRegistry(request DeleteFromRegistryRequest) e
 
 	return nil
 }
+
+type PruneFromRegistryRequest struct {
+	Permissive    bool
+	InputDatabase string
+	Packages      []string
+}
+
+func (r RegistryUpdater) PruneFromRegistry(request PruneFromRegistryRequest) error {
+	db, err := sql.Open("sqlite3", request.InputDatabase)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	dbLoader, err := sqlite.NewSQLLiteLoader(db)
+	if err != nil {
+		return err
+	}
+	if err := dbLoader.Migrate(context.TODO()); err != nil {
+		return err
+	}
+
+	// get all the packages
+	lister := sqlite.NewSQLLiteQuerierFromDb(db)
+	packages, err := lister.ListPackages(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	// make it inexpensive to find packages
+	pkgMap := make(map[string]bool)
+	for _, pkg := range request.Packages {
+		pkgMap[pkg] = true
+	}
+
+	// prune packages from registry
+	for _, pkg := range packages {
+		if _, found := pkgMap[pkg]; !found {
+			remover := sqlite.NewSQLRemoverForPackages(dbLoader, pkg)
+			if err := remover.Remove(); err != nil {
+				err = fmt.Errorf("error deleting packages from database: %s", err)
+				if !request.Permissive {
+					logrus.WithError(err).Fatal("permissive mode disabled")
+					return err
+				}
+				logrus.WithError(err).Warn("permissive mode enabled")
+			}
+		}
+	}
+
+	return nil
+}

test/e2e/opm_test.go

@@ -34,10 +34,12 @@ var (
 	bundleTag3 = rand.String(6)
 	indexTag1  = rand.String(6)
 	indexTag2  = rand.String(6)
+	indexTag3  = rand.String(6)
 
 	bundleImage = "quay.io/olmtest/e2e-bundle"
 	indexImage1 = "quay.io/olmtest/e2e-index:" + indexTag1
 	indexImage2 = "quay.io/olmtest/e2e-index:" + indexTag2
+	indexImage3 = "quay.io/olmtest/e2e-index:" + indexTag3
 )
 
 func inTemporaryBuildContext(f func() error) (rerr error) {
@@ -122,6 +124,24 @@ func buildFromIndexWith(containerTool string) error {
 	return indexAdder.AddToIndex(request)
 }
 
+// TODO(djzager): make this more complete than what should be a simple no-op
+func pruneIndexWith(containerTool string) error {
+	logger := logrus.WithFields(logrus.Fields{"packages": packageName})
+	indexAdder := indexer.NewIndexPruner(containertools.NewContainerTool(containerTool, containertools.NoneTool), logger)
+
+	request := indexer.PruneFromIndexRequest{
+		Generate:          false,
+		FromIndex:         indexImage2,
+		BinarySourceImage: "",
+		OutDockerfile:     "",
+		Tag:               indexImage3,
+		Packages:          []string{packageName},
+		Permissive:        false,
+	}
+
+	return indexAdder.PruneFromIndex(request)
+}
+
 func pushWith(containerTool, image string) error {
 	dockerpush := exec.Command(containerTool, "push", image)
 	return dockerpush.Run()
@@ -216,6 +236,14 @@ var _ = Describe("opm", func() {
 			err = pushWith(containerTool, indexImage2)
 			Expect(err).NotTo(HaveOccurred())
 
+			By("pruning an index")
+			err = pruneIndexWith(containerTool)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("pushing an index")
+			err = pushWith(containerTool, indexImage3)
+			Expect(err).NotTo(HaveOccurred())
+
 			By("exporting an index to disk")
 			err = exportWith(containerTool)
 			Expect(err).NotTo(HaveOccurred())