Merge pull request #413 from kevinrizza/cleanup-command

Bug 1866930: Clean up stranded bundles

Author: openshift-merge-robot

cmd/opm/index/cmd.go

@@ -26,4 +26,5 @@ func AddCommand(parent *cobra.Command) {
 	cmd.AddCommand(newIndexExportCmd())
 	cmd.AddCommand(newIndexPruneCmd())
 	cmd.AddCommand(newIndexDeprecateTruncateCmd())
+	cmd.AddCommand(newIndexPruneStrandedCmd())
 }

cmd/opm/index/prunestranded.go

@@ -0,0 +1,103 @@
+package index
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/operator-framework/operator-registry/pkg/containertools"
+	"github.com/operator-framework/operator-registry/pkg/lib/indexer"
+)
+
+func newIndexPruneStrandedCmd() *cobra.Command {
+	indexCmd := &cobra.Command{
+		Use:   "prune-stranded",
+		Short: "prune an index of stranded bundles",
+		Long:  `prune an index of stranded bundles - bundles that are not associated with a particular package`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runIndexPruneStrandedCmdFunc,
+	}
+
+	indexCmd.Flags().Bool("debug", false, "enable debug logging")
+	indexCmd.Flags().Bool("generate", false, "if enabled, just creates the dockerfile and saves it to local disk")
+	indexCmd.Flags().StringP("out-dockerfile", "d", "", "if generating the dockerfile, this flag is used to (optionally) specify a dockerfile name")
+	indexCmd.Flags().StringP("from-index", "f", "", "index to prune")
+	if err := indexCmd.MarkFlagRequired("from-index"); err != nil {
+		logrus.Panic("Failed to set required `from-index` flag for `index prune-stranded`")
+	}
+	indexCmd.Flags().StringP("binary-image", "i", "", "container image for on-image `opm` command")
+	indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
+	indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
+
+	if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
+		logrus.Panic(err.Error())
+	}
+
+	return indexCmd
+
+}
+
+func runIndexPruneStrandedCmdFunc(cmd *cobra.Command, args []string) error {
+	generate, err := cmd.Flags().GetBool("generate")
+	if err != nil {
+		return err
+	}
+
+	outDockerfile, err := cmd.Flags().GetString("out-dockerfile")
+	if err != nil {
+		return err
+	}
+
+	fromIndex, err := cmd.Flags().GetString("from-index")
+	if err != nil {
+		return err
+	}
+
+	binaryImage, err := cmd.Flags().GetString("binary-image")
+	if err != nil {
+		return err
+	}
+
+	containerTool, err := cmd.Flags().GetString("container-tool")
+	if err != nil {
+		return err
+	}
+
+	if containerTool == "none" {
+		return fmt.Errorf("none is not a valid container-tool for index prune")
+	}
+
+	tag, err := cmd.Flags().GetString("tag")
+	if err != nil {
+		return err
+	}
+
+	logger := logrus.WithFields(logrus.Fields{})
+
+	logger.Info("pruning stranded bundles from the index")
+
+	indexPruner := indexer.NewIndexStrandedPruner(containertools.NewContainerTool(containerTool, containertools.PodmanTool), logger)
+
+	request := indexer.PruneStrandedFromIndexRequest{
+		Generate:          generate,
+		FromIndex:         fromIndex,
+		BinarySourceImage: binaryImage,
+		OutDockerfile:     outDockerfile,
+		Tag:               tag,
+	}
+
+	err = indexPruner.PruneStrandedFromIndex(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/opm/registry/cmd.go

@@ -24,6 +24,7 @@ func NewOpmRegistryCmd() *cobra.Command {
 	rootCmd.AddCommand(newRegistryAddCmd())
 	rootCmd.AddCommand(newRegistryRmCmd())
 	rootCmd.AddCommand(newRegistryPruneCmd())
+	rootCmd.AddCommand(newRegistryPruneStrandedCmd())
 
 	return rootCmd
 }

cmd/opm/registry/prunestranded.go

@@ -0,0 +1,54 @@
+package registry
+
+import (
+	"github.com/operator-framework/operator-registry/pkg/lib/registry"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func newRegistryPruneStrandedCmd() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:   "prune-stranded",
+		Short: "prune an operator registry DB of stranded bundles",
+		Long:  `prune an operator registry DB of stranded bundles - bundles that are not associated with a particular package`,
+
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if debug, _ := cmd.Flags().GetBool("debug"); debug {
+				logrus.SetLevel(logrus.DebugLevel)
+			}
+			return nil
+		},
+
+		RunE: runRegistryPruneStrandedCmdFunc,
+	}
+
+	rootCmd.Flags().Bool("debug", false, "enable debug logging")
+	rootCmd.Flags().StringP("database", "d", "bundles.db", "relative path to database file")
+
+	return rootCmd
+}
+
+func runRegistryPruneStrandedCmdFunc(cmd *cobra.Command, args []string) error {
+	fromFilename, err := cmd.Flags().GetString("database")
+	if err != nil {
+		return err
+	}
+
+	request := registry.PruneStrandedFromRegistryRequest{
+		InputDatabase: fromFilename,
+	}
+
+	logger := logrus.WithFields(logrus.Fields{})
+
+	logger.Info("pruning from the registry")
+
+	registryStrandedPruner := registry.NewRegistryStrandedPruner(logger)
+
+	err = registryStrandedPruner.PruneStrandedFromRegistry(request)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/lib/indexer/indexer.go

@@ -35,16 +35,17 @@ const (
 
 // ImageIndexer is a struct implementation of the Indexer interface
 type ImageIndexer struct {
-	DockerfileGenerator containertools.DockerfileGenerator
-	CommandRunner       containertools.CommandRunner
-	LabelReader         containertools.LabelReader
-	RegistryAdder       registry.RegistryAdder
-	RegistryDeleter     registry.RegistryDeleter
-	RegistryPruner      registry.RegistryPruner
-	RegistryDeprecator  registry.RegistryDeprecator
-	BuildTool           containertools.ContainerTool
-	PullTool            containertools.ContainerTool
-	Logger              *logrus.Entry
+	DockerfileGenerator    containertools.DockerfileGenerator
+	CommandRunner          containertools.CommandRunner
+	LabelReader            containertools.LabelReader
+	RegistryAdder          registry.RegistryAdder
+	RegistryDeleter        registry.RegistryDeleter
+	RegistryPruner         registry.RegistryPruner
+	RegistryStrandedPruner registry.RegistryStrandedPruner
+	RegistryDeprecator     registry.RegistryDeprecator
+	BuildTool              containertools.ContainerTool
+	PullTool               containertools.ContainerTool
+	Logger                 *logrus.Entry
 }
 
 // AddToIndexRequest defines the parameters to send to the AddToIndex API
@@ -168,6 +169,59 @@ func (i ImageIndexer) DeleteFromIndex(request DeleteFromIndexRequest) error {
 	return nil
 }
 
+// PruneStrandedFromIndexRequest defines the parameters to send to the PruneStrandedFromIndex API
+type PruneStrandedFromIndexRequest struct {
+	Generate          bool
+	BinarySourceImage string
+	FromIndex         string
+	OutDockerfile     string
+	Tag               string
+}
+
+// PruneStrandedFromIndex is an aggregate API used to generate a registry index image
+// that has removed stranded bundles from the index
+func (i ImageIndexer) PruneStrandedFromIndex(request PruneStrandedFromIndexRequest) error {
+	buildDir, outDockerfile, cleanup, err := buildContext(request.Generate, request.OutDockerfile)
+	defer cleanup()
+	if err != nil {
+		return err
+	}
+
+	databasePath, err := i.extractDatabase(buildDir, request.FromIndex)
+	if err != nil {
+		return err
+	}
+
+	// Run opm registry prune-stranded on the database
+	pruneStrandedFromRegistryReq := registry.PruneStrandedFromRegistryRequest{
+		InputDatabase: databasePath,
+	}
+
+	// Delete the stranded bundles from the registry
+	err = i.RegistryStrandedPruner.PruneStrandedFromRegistry(pruneStrandedFromRegistryReq)
+	if err != nil {
+		return err
+	}
+
+	// generate the dockerfile
+	dockerfile := i.DockerfileGenerator.GenerateIndexDockerfile(request.BinarySourceImage, databasePath)
+	err = write(dockerfile, outDockerfile, i.Logger)
+	if err != nil {
+		return err
+	}
+
+	if request.Generate {
+		return nil
+	}
+
+	// build the dockerfile
+	err = build(outDockerfile, request.Tag, i.CommandRunner, i.Logger)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // PruneFromIndexRequest defines the parameters to send to the PruneFromIndex API
 type PruneFromIndexRequest struct {
 	Generate          bool

pkg/lib/indexer/interfaces.go

@@ -64,6 +64,23 @@ func NewIndexExporter(containerTool containertools.ContainerTool, logger *logrus
 	}
 }
 
+// IndexStrandedPruner prunes operators out of an index
+type IndexStrandedPruner interface {
+	PruneStrandedFromIndex(PruneStrandedFromIndexRequest) error
+}
+
+func NewIndexStrandedPruner(containerTool containertools.ContainerTool, logger *logrus.Entry) IndexStrandedPruner {
+	return ImageIndexer{
+		DockerfileGenerator:    containertools.NewDockerfileGenerator(logger),
+		CommandRunner:          containertools.NewCommandRunner(containerTool, logger),
+		LabelReader:            containertools.NewLabelReader(containerTool, logger),
+		RegistryStrandedPruner: registry.NewRegistryStrandedPruner(logger),
+		BuildTool:              containerTool,
+		PullTool:               containerTool,
+		Logger:                 logger,
+	}
+}
+
 // IndexPruner prunes operators out of an index
 type IndexPruner interface {
 	PruneFromIndex(PruneFromIndexRequest) error

pkg/lib/registry/interfaces.go

@@ -27,6 +27,16 @@ func NewRegistryDeleter(logger *logrus.Entry) RegistryDeleter {
 	}
 }
 
+type RegistryStrandedPruner interface {
+	PruneStrandedFromRegistry(PruneStrandedFromRegistryRequest) error
+}
+
+func NewRegistryStrandedPruner(logger *logrus.Entry) RegistryStrandedPruner {
+	return RegistryUpdater{
+		Logger: logger,
+	}
+}
+
 type RegistryPruner interface {
 	PruneFromRegistry(PruneFromRegistryRequest) error
 }

pkg/lib/registry/registry.go

@@ -174,6 +174,40 @@ func (r RegistryUpdater) DeleteFromRegistry(request DeleteFromRegistryRequest) e
 		}
 	}
 
+	// remove any stranded bundles from the database
+	// TODO: This is unnecessary if the db schema can prevent this orphaned data from existing
+	remover := sqlite.NewSQLStrandedBundleRemover(dbLoader)
+	if err := remover.Remove(); err != nil {
+		return fmt.Errorf("error removing stranded packages from database: %s", err)
+	}
+
+	return nil
+}
+
+type PruneStrandedFromRegistryRequest struct {
+	InputDatabase string
+}
+
+func (r RegistryUpdater) PruneStrandedFromRegistry(request PruneStrandedFromRegistryRequest) error {
+	db, err := sql.Open("sqlite3", request.InputDatabase)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	dbLoader, err := sqlite.NewSQLLiteLoader(db)
+	if err != nil {
+		return err
+	}
+	if err := dbLoader.Migrate(context.TODO()); err != nil {
+		return err
+	}
+
+	remover := sqlite.NewSQLStrandedBundleRemover(dbLoader)
+	if err := remover.Remove(); err != nil {
+		return fmt.Errorf("error removing stranded packages from database: %s", err)
+	}
+
 	return nil
 }
 

pkg/registry/interface.go

@@ -12,6 +12,7 @@ type Load interface {
 	AddPackageChannels(manifest PackageManifest) error
 	AddBundlePackageChannels(manifest PackageManifest, bundle *Bundle) error
 	RemovePackage(packageName string) error
+	RemoveStrandedBundles() ([]string, error)
 	DeprecateBundle(path string) error
 	ClearNonHeadBundles() error
 }