feat(opm): add unprivileged index extract

index extract, by default or with the "none" containertool option,
will pull and extract indexes and bundles without a deamon

Author: ecordell

cmd/opm/index/export.go

@@ -1,7 +1,6 @@
 package index
 
 import (
-	"fmt"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"k8s.io/kubectl/pkg/util/templates"
@@ -46,7 +45,7 @@ func newIndexExportCmd() *cobra.Command {
 		logrus.Panic("Failed to set required `package` flag for `index export`")
 	}
 	indexCmd.Flags().StringP("download-folder", "f", "downloaded", "directory where downloaded operator bundle(s) will be stored")
-	indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [none, docker, podman]")
+	indexCmd.Flags().StringP("container-tool", "c", "none", "tool to interact with container images (save, build, etc.). One of: [none, docker, podman]")
 	if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
 		logrus.Panic(err.Error())
 	}
@@ -76,21 +75,17 @@ func runIndexExportCmdFunc(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if containerTool == "none" {
-		return fmt.Errorf("none is not a valid container-tool for index add")
-	}
-
 	logger := logrus.WithFields(logrus.Fields{"index": index, "package": packageName})
 
 	logger.Info("export from the index")
 
-	indexExporter := indexer.NewIndexExporter(containertools.NewContainerTool(containerTool, containertools.PodmanTool), logger)
+	indexExporter := indexer.NewIndexExporter(containertools.NewContainerTool(containerTool, containertools.NoneTool), logger)
 
 	request := indexer.ExportFromIndexRequest{
 		Index:         index,
 		Package:       packageName,
 		DownloadPath:  downloadPath,
-		ContainerTool: containerTool,
+		ContainerTool: containertools.NewContainerTool(containerTool, containertools.NoneTool),
 	}
 
 	err = indexExporter.ExportFromIndex(request)

pkg/image/containerdregistry/registry.go

@@ -2,7 +2,9 @@ package containerdregistry
 
 import (
 	"archive/tar"
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"github.com/containerd/containerd/archive"
 	"github.com/containerd/containerd/archive/compression"
@@ -14,6 +16,7 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
 	"io"
+	"io/ioutil"
 	"os"
 
 	"github.com/operator-framework/operator-registry/pkg/image"
@@ -93,6 +96,52 @@ func (r *Registry) Unpack(ctx context.Context, ref image.Reference, dir string)
 	return nil
 }
 
+// Labels gets the labels for an image reference.
+func (r *Registry) Labels(ctx context.Context, ref image.Reference) (map[string]string, error) {
+	// Set the default namespace if unset
+	ctx = ensureNamespace(ctx)
+	tmpDir, err := ioutil.TempDir("./", "bundle_tmp")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	img, err := r.Images().Get(ctx, ref.String())
+	if err != nil {
+		return nil, err
+	}
+
+	manifest, err := images.Manifest(ctx, r.Content(), img.Target, r.platform)
+	if err != nil {
+		return nil, err
+	}
+
+	ra, err := r.Content().ReaderAt(ctx, manifest.Config)
+	if err != nil {
+		return nil, err
+	}
+	defer ra.Close()
+
+	decompressed, err := compression.DecompressStream(io.NewSectionReader(ra, 0, ra.Size()))
+	if err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	if _, err := io.Copy(&buf, decompressed); err != nil {
+		return nil, err
+	}
+	r.log.Warn(buf.String())
+
+	var imageConfig ocispec.Image
+
+	if err := json.Unmarshal(buf.Bytes(), &imageConfig); err != nil {
+		return nil, err
+	}
+
+	return imageConfig.Config.Labels, nil
+}
+
+
 // Destroy cleans up the on-disk boltdb file and other cache files, unless preserve cache is true
 func (r *Registry) Destroy() (err error) {
 	return r.destroy()

pkg/image/execregistry/registry.go

@@ -38,6 +38,14 @@ func (r *Registry) Unpack(ctx context.Context, ref image.Reference, dir string)
 	}.GetImageData(ref.String(), dir)
 }
 
+// Labels gets the labels for an image reference.
+func (r *Registry) Labels(ctx context.Context, ref image.Reference) (map[string]string, error) {
+	return containertools.ImageLabelReader{
+		Cmd: r.cmd,
+		Logger: r.log,
+	}.GetLabelsFromImage(ref.String())
+}
+
 // Destroy is no-op for exec tools
 func (r *Registry) Destroy() error {
 	return nil

pkg/image/registry.go

@@ -18,6 +18,9 @@ type Registry interface {
 	// If the referenced image does not exist in the registry, an error is returned.
 	Unpack(ctx context.Context, ref Reference, dir string) error
 
+	// Labels gets the labels for an image that is already stored.
+	Labels(ctx context.Context, ref Reference) (map[string]string, error)
+
 	// Destroy cleans up any on-disk resources used to track images
 	Destroy() error
 
@@ -26,3 +29,4 @@ type Registry interface {
 	// If it exists, it's used as the base image.
 	// Pack(ctx context.Context, ref Reference, from io.Reader) (next string, err error)
 }
+

pkg/lib/bundle/exporter.go

@@ -1,6 +1,9 @@
 package bundle
 
 import (
+	"context"
+	"github.com/operator-framework/operator-registry/pkg/image"
+	"github.com/operator-framework/operator-registry/pkg/image/execregistry"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -9,6 +12,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/operator-framework/operator-registry/pkg/containertools"
+	"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
 )
 
 // BundleExporter exports the manifests of a bundle image into a directory
@@ -18,11 +22,11 @@ type BundleExporter struct {
 	containerTool containertools.ContainerTool
 }
 
-func NewSQLExporterForBundle(image, directory, containerTool string) *BundleExporter {
+func NewSQLExporterForBundle(image, directory string, containerTool containertools.ContainerTool) *BundleExporter {
 	return &BundleExporter{
 		image:         image,
 		directory:     directory,
-		containerTool: containertools.NewContainerTool(containerTool, containertools.NoneTool),
+		containerTool: containerTool,
 	}
 }
 
@@ -36,13 +40,33 @@ func (i *BundleExporter) Export() error {
 	}
 	defer os.RemoveAll(tmpDir)
 
-	// Pull the image and get the manifests
-	reader := containertools.NewImageReader(i.containerTool, log)
+	var reg image.Registry
+	var rerr error
+	switch i.containerTool {
+	case containertools.NoneTool:
+		reg, rerr = containerdregistry.NewRegistry(containerdregistry.WithLog(log))
+	case containertools.PodmanTool:
+		fallthrough
+	case containertools.DockerTool:
+		reg, rerr = execregistry.NewRegistry(i.containerTool, log)
+	}
+	if rerr != nil {
+		return rerr
+	}
+	defer func() {
+		if err := reg.Destroy(); err != nil {
+			log.WithError(err).Warn("error destroying local cache")
+		}
+	}()
 
-	err = reader.GetImageData(i.image, tmpDir)
-	if err != nil {
+	if err := reg.Pull(context.TODO(), image.SimpleReference(i.image)); err != nil {
 		return err
 	}
+
+	if err := reg.Unpack(context.TODO(), image.SimpleReference(i.image), tmpDir); err != nil {
+		return err
+	}
+
 	if err := os.MkdirAll(i.directory, 0777); err != nil {
 		return err
 	}

pkg/lib/indexer/indexer.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"github.com/operator-framework/operator-registry/pkg/image"
+	"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
+	"github.com/operator-framework/operator-registry/pkg/image/execregistry"
 	"io"
 	"io/ioutil"
 	"os"
@@ -210,19 +213,43 @@ func (i ImageIndexer) getDatabaseFile(workingDir, fromIndex string) (string, err
 	// Pull the fromIndex
 	i.Logger.Infof("Pulling previous image %s to get metadata", fromIndex)
 
+	var reg image.Registry
+	var rerr error
+	switch i.ContainerTool {
+	case containertools.NoneTool:
+		reg, rerr = containerdregistry.NewRegistry(containerdregistry.WithLog(i.Logger))
+	case containertools.PodmanTool:
+		fallthrough
+	case containertools.DockerTool:
+		reg, rerr = execregistry.NewRegistry(i.ContainerTool, i.Logger)
+	}
+	if rerr != nil {
+		return "", rerr
+	}
+	defer func() {
+		if err := reg.Destroy(); err != nil {
+			i.Logger.WithError(err).Warn("error destroying local cache")
+		}
+	}()
+
+	imageRef := image.SimpleReference(fromIndex)
+
+	if err := reg.Pull(context.TODO(), imageRef); err != nil {
+		return "", err
+	}
+
 	// Get the old index image's dbLocationLabel to find this path
-	labels, err := i.LabelReader.GetLabelsFromImage(fromIndex)
+	labels, err := reg.Labels(context.TODO(), imageRef)
 	if err != nil {
 		return "", err
 	}
+
 	dbLocation, ok := labels[containertools.DbLocationLabel]
 	if !ok {
-		return "", fmt.Errorf("Index image %s missing label %s", fromIndex, containertools.DbLocationLabel)
+		return "", fmt.Errorf("index image %s missing label %s", fromIndex, containertools.DbLocationLabel)
 	}
 
-	// extract the database to the working directory
-	err = i.ImageReader.GetImageData(fromIndex, workingDir)
-	if err != nil {
+	if err := reg.Unpack(context.TODO(), imageRef, workingDir); err != nil {
 		return "", err
 	}
 
@@ -341,7 +368,7 @@ type ExportFromIndexRequest struct {
 	Index         string
 	Package       string
 	DownloadPath  string
-	ContainerTool string
+	ContainerTool containertools.ContainerTool
 }
 
 // ExportFromIndex is an aggregate API used to specify operators from
@@ -355,7 +382,7 @@ func (i ImageIndexer) ExportFromIndex(request ExportFromIndexRequest) error {
 	defer os.RemoveAll(workingDir)
 
 	// extract the index database to the file
-	databaseFile, err := i.WriteIndexDBFile(request.Index, workingDir, defaultDatabaseFile)
+	databaseFile, err := i.getDatabaseFile(workingDir, request.Index)
 	if err != nil {
 		return err
 	}
@@ -411,32 +438,6 @@ func (i ImageIndexer) ExportFromIndex(request ExportFromIndexRequest) error {
 	return utilerrors.NewAggregate(errs)
 }
 
-func (i ImageIndexer) WriteIndexDBFile(index, workingDir, databaseFile string) (string, error) {
-	if index != "" {
-		i.Logger.Infof("Pulling previous image %s to get metadata", index)
-
-		// Get the old index image's dbLocationLabel to find this path
-		labels, err := i.LabelReader.GetLabelsFromImage(index)
-		if err != nil {
-			return "", err
-		}
-		if dbLocation, ok := labels[containertools.DbLocationLabel]; ok {
-			i.Logger.Infof("Previous db location %s", dbLocation)
-
-			// extract the database to the file
-			err = i.ImageReader.GetImageData(index, workingDir)
-			if err != nil {
-				return "", err
-			}
-
-			databaseFile = path.Join(workingDir, dbLocation)
-		}
-	} else {
-		databaseFile = path.Join(workingDir, databaseFile)
-	}
-	return databaseFile, nil
-}
-
 func getBundlesToExport(dbQuerier pregistry.Query, packageName string) ([]string, error) {
 	bundles, err := dbQuerier.GetBundlePathsForPackage(context.TODO(), packageName)
 	if err != nil {

test/e2e/opm_test.go

@@ -148,7 +148,7 @@ func exportWith(containerTool string) error {
 		Index:         indexImage2,
 		Package:       packageName,
 		DownloadPath:  "downloaded",
-		ContainerTool: containerTool,
+		ContainerTool: containertools.NewContainerTool(containerTool, containertools.NoneTool),
 	}
 
 	return indexExporter.ExportFromIndex(request)
@@ -223,6 +223,18 @@ var _ = Describe("opm", func() {
 			By("loading manifests from a directory")
 			err = initialize()
 			Expect(err).NotTo(HaveOccurred())
+
+			// clean and try again with containerd
+			err = os.RemoveAll("downloaded")
+			Expect(err).NotTo(HaveOccurred())
+
+			By("exporting an index to disk with containerd")
+			err = exportWith(containertools.NoneTool.String())
+			Expect(err).NotTo(HaveOccurred())
+
+			By("loading manifests from a containerd-extracted directory")
+			err = initialize()
+			Expect(err).NotTo(HaveOccurred())
 		})
 	}