Skip to content

Commit 3389781

Browse files
gallettilancegallettilance
committedJan 24, 2020
Adding ability to export package from index
·
v1.51.0goreleaser-all-os
1 parent cb8d55a commit 3389781

20 files changed

+856
-75
lines changed
 

‎Makefile

+2
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,5 @@ generate-fakes:
4444
clean:
4545
@rm -rf ./bin
4646

47+
opm-test:
48+
$(shell ./opm-test.sh || echo "opm-test FAIL")

‎bundles/etcd.0.9.2/manifests/etcdoperator.v0.9.2.clusterserviceversion.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,23 @@ spec:
164164
fieldRef:
165165
fieldPath: metadata.name
166166
customresourcedefinitions:
167+
required:
168+
- name: etcdclusters.etcd.database.coreos.com
169+
version: v1beta2
170+
kind: EtcdCluster
171+
displayName: etcd Cluster
172+
description: Represents a cluster of etcd nodes.
173+
resources:
174+
- kind: Service
175+
version: v1
176+
- kind: Pod
177+
version: v1
178+
specDescriptors:
179+
- description: The desired number of member Pods for the etcd cluster.
180+
displayName: Size
181+
path: size
182+
x-descriptors:
183+
- 'urn:alm:descriptor:com.tectonic.ui:podCount'
167184
owned:
168185
- name: etcdclusters.etcd.database.coreos.com
169186
version: v1beta2

‎cmd/opm/index/cmd.go

+1
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,5 @@ func AddCommand(parent *cobra.Command) {
2323
parent.AddCommand(cmd)
2424
cmd.AddCommand(newIndexDeleteCmd())
2525
addIndexAddCmd(cmd)
26+
cmd.AddCommand(newIndexExportCmd())
2627
}

‎cmd/opm/index/export.go

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
package index
2+
3+
import (
4+
"github.com/sirupsen/logrus"
5+
"github.com/spf13/cobra"
6+
"k8s.io/kubectl/pkg/util/templates"
7+
8+
"github.com/operator-framework/operator-registry/pkg/lib/indexer"
9+
)
10+
11+
var exportLong = templates.LongDesc(`
12+
Export an operator from an index image into the appregistry format.
13+
14+
This command will take an index image (specified by the --index option), parse it for the given operator (set by
15+
the --operator option) and export the operator metadata into an appregistry compliant format (a package.yaml file).
16+
This command requires access to docker or podman to complete successfully.
17+
18+
Note: the appregistry format is being deprecated in favor of the new index image and image bundle format.
19+
`)
20+
21+
func newIndexExportCmd() *cobra.Command {
22+
indexCmd := &cobra.Command{
23+
Use: "export",
24+
Short: "Export an operator from an index into the appregistry format",
25+
Long: exportLong,
26+
27+
PreRunE: func(cmd *cobra.Command, args []string) error {
28+
if debug, _ := cmd.Flags().GetBool("debug"); debug {
29+
logrus.SetLevel(logrus.DebugLevel)
30+
}
31+
return nil
32+
},
33+
34+
RunE: runIndexExportCmdFunc,
35+
}
36+
37+
indexCmd.Flags().Bool("debug", false, "enable debug logging")
38+
indexCmd.Flags().StringP("index", "i", "", "index to get package from")
39+
if err := indexCmd.MarkFlagRequired("index"); err != nil {
40+
logrus.Panic("Failed to set required `index` flag for `index export`")
41+
}
42+
indexCmd.Flags().StringP("package", "o", "", "the package to export")
43+
if err := indexCmd.MarkFlagRequired("package"); err != nil {
44+
logrus.Panic("Failed to set required `package` flag for `index export`")
45+
}
46+
indexCmd.Flags().StringP("download-folder", "f", "downloaded", "directory where downloaded operator bundle(s) will be stored")
47+
indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
48+
if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
49+
logrus.Panic(err.Error())
50+
}
51+
52+
return indexCmd
53+
54+
}
55+
56+
func runIndexExportCmdFunc(cmd *cobra.Command, args []string) error {
57+
index, err := cmd.Flags().GetString("index")
58+
if err != nil {
59+
return err
60+
}
61+
62+
packageName, err := cmd.Flags().GetString("package")
63+
if err != nil {
64+
return err
65+
}
66+
67+
downloadPath, err := cmd.Flags().GetString("download-folder")
68+
if err != nil {
69+
return err
70+
}
71+
72+
containerTool, err := cmd.Flags().GetString("container-tool")
73+
if err != nil {
74+
return err
75+
}
76+
77+
logger := logrus.WithFields(logrus.Fields{"index": index, "package": packageName})
78+
79+
logger.Info("export from the index")
80+
81+
indexExporter := indexer.NewIndexExporter(containerTool, logger)
82+
83+
request := indexer.ExportFromIndexRequest{
84+
Index: index,
85+
Package: packageName,
86+
DownloadPath: downloadPath,
87+
ContainerTool: containerTool,
88+
}
89+
90+
err = indexExporter.ExportFromIndex(request)
91+
if err != nil {
92+
return err
93+
}
94+
95+
return nil
96+
}

‎docs/design/opm-tooling.md

+35
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,41 @@ Like `opm registry rm`, this command will remove all versions an entire operator
9393

9494
This will result in the tagged container image `quay.io/operator-framework/monitoring-index:1.0.2` with a registry that no longer contains the `prometheus` operator at all.
9595

96+
#### export
97+
98+
`opm index export` will export a package from an index image into a directory. The format of this directory will match the appregistry manifest format: containing all versions of the package in the index along with a `package.yaml` file. This command takes an `--index` flag that points to an index image, a `--package` flag that states a package name, an optional `--download-folder` as the export location (default is `./downloaded`), and just as the other index commands it takes a `--container-tool` flag. Ex:
99+
100+
`opm index export --index="quay.io/operator-framework/monitoring:1.0.0" --package="prometheus" -c="podman"`
101+
102+
This will result in the following `downloaded` folder directory structure:
103+
104+
```bash
105+
downloaded
106+
├── 0.14.0
107+
│ ├── alertmanager.crd.yaml
108+
│ ├── prometheus.crd.yaml
109+
│ ├── prometheusoperator.0.14.0.clusterserviceversion.yaml
110+
│ ├── prometheusrule.crd.yaml
111+
│ └── servicemonitor.crd.yaml
112+
├── 0.15.0
113+
│ ├── alertmanager.crd.yaml
114+
│ ├── prometheus.crd.yaml
115+
│ ├── prometheusoperator.0.15.0.clusterserviceversion.yaml
116+
│ ├── prometheusrule.crd.yaml
117+
│ └── servicemonitor.crd.yaml
118+
├── 0.22.2
119+
│ ├── alertmanager.crd.yaml
120+
│ ├── prometheus.crd.yaml
121+
│ ├── prometheusoperator.0.22.2.clusterserviceversion.yaml
122+
│ ├── prometheusrule.crd.yaml
123+
│ └── servicemonitor.crd.yaml
124+
└── package.yaml
125+
```
126+
127+
which can be pushed to appregistry.
128+
129+
**Note**: the appregistry format is being deprecated in favor of the new index image and image bundle format.
130+
96131
### Container Tooling
97132

98133
Of note, many of these commands require some form of shelling to common container tooling (in the general case at least to pull the bundle image and extract the contents to update the registry). By default, the container tool that `opm` shells to is [podman](https://podman.io/). However, we also support overriding this via the `--container-tool` flag in all of these commands:

‎go.sum

-2
Original file line numberDiff line numberDiff line change
@@ -514,8 +514,6 @@ github.com/openshift/api v3.9.1-0.20190924102528-32369d4db2ad+incompatible/go.mo
514514
github.com/openshift/client-go v0.0.0-20190923180330-3b6373338c9b/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk=
515515
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
516516
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
517-
github.com/operator-framework/api v0.0.0-20191127212340-9066a6e95573 h1:bMO43IWWPM3HCGIiuM/GyXjtSJWsrhOlzUpZMesVUw0=
518-
github.com/operator-framework/api v0.0.0-20191127212340-9066a6e95573/go.mod h1:S5IdlJvmKkF84K2tBvsrqJbI2FVy03P88R75snpRxJo=
519517
github.com/operator-framework/api v0.0.0-20200120235816-80fd2f1a09c9 h1:HfxMEPJ0djo/RNfrmli3kI2oKS6IeuIZWu1Q5Rewt/o=
520518
github.com/operator-framework/api v0.0.0-20200120235816-80fd2f1a09c9/go.mod h1:S5IdlJvmKkF84K2tBvsrqJbI2FVy03P88R75snpRxJo=
521519
github.com/operator-framework/operator-lifecycle-manager v0.0.0-20191115003340-16619cd27fa5 h1:rjaihxY50c5C+kbQIK4s36R8zxByATYrgRbua4eiG6o=

‎opm-test.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
set -euxo pipefail
3+
4+
echo "Setting Variables"
5+
read -p "Bunlde image path: " bundleImage
6+
read -p "Index image path: " indexImage
7+
read -p "docker or podman: " containerTool
8+
9+
echo "building opm"
10+
go build -mod=vendor -o bin/opm ./cmd/opm
11+
echo "building prometheus bundles"
12+
./bin/opm alpha bundle build --directory manifests/prometheus/0.14.0 --tag $bundleImage:0.14.0 --package prometheus --channels preview --default preview
13+
./bin/opm alpha bundle build --directory manifests/prometheus/0.15.0 --tag $bundleImage:0.15.0 --package prometheus --channels preview --default preview
14+
./bin/opm alpha bundle build --directory manifests/prometheus/0.22.2 --tag $bundleImage:0.22.2 --package prometheus --channels preview --default preview
15+
echo "pushing prometheus bundles"
16+
docker push $bundleImage:0.14.0
17+
docker push $bundleImage:0.15.0
18+
docker push $bundleImage:0.22.2
19+
echo "building index image with prometheus"
20+
./bin/opm index add -b="$bundleImage:0.14.0,$bundleImage:0.15.0,$bundleImage:0.22.2" -t "$indexImage" -c="$containerTool"
21+
echo "pushing index image"
22+
docker push $indexImage
23+
echo "sleep for 1min before pulling image"
24+
sleep 1m
25+
echo "exporting from index"
26+
./bin/opm index export -i="$indexImage" -o="prometheus" -c="$containerTool"

‎pkg/lib/bundle/exporter.go

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package bundle
2+
3+
import (
4+
"io/ioutil"
5+
"os"
6+
"path/filepath"
7+
8+
"github.com/otiai10/copy"
9+
"github.com/sirupsen/logrus"
10+
11+
"github.com/operator-framework/operator-registry/pkg/containertools"
12+
)
13+
14+
// BundleExporter exports the manifests of a bundle image into a directory
15+
type BundleExporter struct {
16+
image string
17+
directory string
18+
containerTool string
19+
}
20+
21+
func NewSQLExporterForBundle(image, directory, containerTool string) *BundleExporter {
22+
return &BundleExporter{
23+
image: image,
24+
directory: directory,
25+
containerTool: containerTool,
26+
}
27+
}
28+
29+
func (i *BundleExporter) Export() error {
30+
31+
log := logrus.WithField("img", i.image)
32+
33+
tmpDir, err := ioutil.TempDir("./", "bundle_tmp")
34+
if err != nil {
35+
return err
36+
}
37+
defer os.RemoveAll(tmpDir)
38+
39+
// Pull the image and get the manifests
40+
reader := containertools.NewImageReader(i.containerTool, log)
41+
42+
err = reader.GetImageData(i.image, tmpDir)
43+
if err != nil {
44+
return err
45+
}
46+
if err := os.MkdirAll(i.directory, 0777); err != nil {
47+
return err
48+
}
49+
50+
return copy.Copy(filepath.Join(tmpDir, "manifests"), i.directory)
51+
}

‎pkg/lib/indexer/indexer.go

+194-26
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,54 @@
11
package indexer
22

33
import (
4+
"context"
5+
"database/sql"
6+
"fmt"
47
"io"
58
"io/ioutil"
69
"os"
710
"path"
11+
"path/filepath"
812

913
"github.com/operator-framework/operator-registry/pkg/containertools"
14+
"github.com/operator-framework/operator-registry/pkg/lib/bundle"
1015
"github.com/operator-framework/operator-registry/pkg/lib/registry"
16+
pregistry "github.com/operator-framework/operator-registry/pkg/registry"
17+
"github.com/operator-framework/operator-registry/pkg/sqlite"
1118

1219
"github.com/sirupsen/logrus"
20+
"gopkg.in/yaml.v2"
21+
utilerrors "k8s.io/apimachinery/pkg/util/errors"
1322
)
1423

1524
const (
1625
defaultDockerfileName = "index.Dockerfile"
17-
defaultImageTag = "operator-registry-index:latest"
26+
defaultImageTag = "operator-registry-index:latest"
1827
defaultDatabaseFolder = "database"
19-
defaultDatabaseFile = "index.db"
28+
defaultDatabaseFile = "index.db"
2029
)
2130

2231
// ImageIndexer is a struct implementation of the Indexer interface
2332
type ImageIndexer struct {
2433
DockerfileGenerator containertools.DockerfileGenerator
25-
CommandRunner containertools.CommandRunner
26-
LabelReader containertools.LabelReader
27-
ImageReader containertools.ImageReader
28-
RegistryAdder registry.RegistryAdder
29-
RegistryDeleter registry.RegistryDeleter
30-
ContainerTool string
31-
Logger *logrus.Entry
34+
CommandRunner containertools.CommandRunner
35+
LabelReader containertools.LabelReader
36+
ImageReader containertools.ImageReader
37+
RegistryAdder registry.RegistryAdder
38+
RegistryDeleter registry.RegistryDeleter
39+
ContainerTool string
40+
Logger *logrus.Entry
3241
}
3342

3443
// AddToIndexRequest defines the parameters to send to the AddToIndex API
3544
type AddToIndexRequest struct {
36-
Generate bool
37-
Permissive bool
45+
Generate bool
46+
Permissive bool
3847
BinarySourceImage string
39-
FromIndex string
40-
OutDockerfile string
41-
Bundles []string
42-
Tag string
48+
FromIndex string
49+
OutDockerfile string
50+
Bundles []string
51+
Tag string
4352
}
4453

4554
// AddToIndex is an aggregate API used to generate a registry index image with additional bundles
@@ -77,9 +86,9 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
7786

7887
// Run opm registry add on the database
7988
addToRegistryReq := registry.AddToRegistryRequest{
80-
Bundles: request.Bundles,
89+
Bundles: request.Bundles,
8190
InputDatabase: databaseFile,
82-
Permissive: request.Permissive,
91+
Permissive: request.Permissive,
8392
ContainerTool: i.ContainerTool,
8493
}
8594

@@ -107,16 +116,16 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
107116

108117
// DeleteFromIndexRequest defines the parameters to send to the DeleteFromIndex API
109118
type DeleteFromIndexRequest struct {
110-
Generate bool
111-
Permissive bool
119+
Generate bool
120+
Permissive bool
112121
BinarySourceImage string
113-
FromIndex string
114-
OutDockerfile string
115-
Tag string
116-
Operators []string
122+
FromIndex string
123+
OutDockerfile string
124+
Tag string
125+
Operators []string
117126
}
118127

119-
// DeleteFromIndex is an aggregate API used to generate a registry index image
128+
// DeleteFromIndex is an aggregate API used to generate a registry index image
120129
// without specific operators
121130
func (i ImageIndexer) DeleteFromIndex(request DeleteFromIndexRequest) error {
122131
databaseFile := defaultDatabaseFile
@@ -154,9 +163,9 @@ func (i ImageIndexer) DeleteFromIndex(request DeleteFromIndexRequest) error {
154163

155164
// Run opm registry add on the database
156165
deleteFromRegistryReq := registry.DeleteFromRegistryRequest{
157-
Packages: request.Operators,
166+
Packages: request.Operators,
158167
InputDatabase: databaseFile,
159-
Permissive: request.Permissive,
168+
Permissive: request.Permissive,
160169
}
161170

162171
// Add the bundles to the registry
@@ -283,3 +292,162 @@ func write(dockerfileText, outDockerfile string, logger *logrus.Entry) error {
283292

284293
return nil
285294
}
295+
296+
// ExportFromIndexRequest defines the parameters to send to the ExportFromIndex API
297+
type ExportFromIndexRequest struct {
298+
Index string
299+
Package string
300+
DownloadPath string
301+
ContainerTool string
302+
}
303+
304+
// ExportFromIndex is an aggregate API used to specify operators from
305+
// an index image
306+
func (i ImageIndexer) ExportFromIndex(request ExportFromIndexRequest) error {
307+
databaseFile := defaultDatabaseFile
308+
309+
// set a temp directory
310+
workingDir, err := ioutil.TempDir("./", "index_tmp")
311+
if err != nil {
312+
return err
313+
}
314+
defer os.RemoveAll(workingDir)
315+
316+
db, err := sql.Open("sqlite3", databaseFile)
317+
if err != nil {
318+
return err
319+
}
320+
defer db.Close()
321+
322+
dbQuerier := sqlite.NewSQLLiteQuerierFromDb(db)
323+
if err != nil {
324+
return err
325+
}
326+
327+
// extract the index database to the file
328+
databaseFile, err = i.WriteIndexDBFile(request.Index, workingDir, databaseFile)
329+
if err != nil {
330+
return err
331+
}
332+
333+
bundles, err := getBundlesToExport(dbQuerier, request.Package)
334+
if err != nil {
335+
return err
336+
}
337+
i.Logger.Infof("Preparing to pull bundles %+q", bundles)
338+
339+
// Creating downloadPath dir
340+
if err := os.MkdirAll(request.DownloadPath, 0777); err != nil {
341+
return err
342+
}
343+
344+
var errs []error
345+
for _, bundleImage := range bundles {
346+
// try to name the folder
347+
folderName, err := dbQuerier.GetBundleVersion(context.TODO(), bundleImage)
348+
if err != nil {
349+
return err
350+
}
351+
if folderName == "" {
352+
// operator-registry does not care about the folder name
353+
folderName = bundleImage
354+
}
355+
exporter := bundle.NewSQLExporterForBundle(bundleImage, filepath.Join(request.DownloadPath, folderName), request.ContainerTool)
356+
if err := exporter.Export(); err != nil {
357+
err = fmt.Errorf("error exporting bundle from image: %s", err)
358+
errs = append(errs, err)
359+
}
360+
}
361+
if err != nil {
362+
errs = append(errs, err)
363+
return utilerrors.NewAggregate(errs)
364+
}
365+
366+
err = generatePackageYaml(dbQuerier, request.Package, request.DownloadPath)
367+
if err != nil {
368+
errs = append(errs, err)
369+
}
370+
return utilerrors.NewAggregate(errs)
371+
}
372+
373+
func (i ImageIndexer) WriteIndexDBFile(index, workingDir, databaseFile string) (string, error) {
374+
if index != "" {
375+
i.Logger.Infof("Pulling previous image %s to get metadata", index)
376+
377+
// Get the old index image's dbLocationLabel to find this path
378+
labels, err := i.LabelReader.GetLabelsFromImage(index)
379+
if err != nil {
380+
return "", err
381+
}
382+
if dbLocation, ok := labels[containertools.DbLocationLabel]; ok {
383+
i.Logger.Infof("Previous db location %s", dbLocation)
384+
385+
// extract the database to the file
386+
err = i.ImageReader.GetImageData(index, workingDir)
387+
if err != nil {
388+
return "", err
389+
}
390+
391+
databaseFile = path.Join(workingDir, dbLocation)
392+
}
393+
} else {
394+
databaseFile = path.Join(workingDir, databaseFile)
395+
}
396+
return databaseFile, nil
397+
}
398+
399+
func getBundlesToExport(dbQuerier pregistry.Query, packageName string) ([]string, error) {
400+
bundles, err := dbQuerier.GetBundlePathsForPackage(context.TODO(), packageName)
401+
if err != nil {
402+
return nil, err
403+
}
404+
return bundles, nil
405+
}
406+
407+
func generatePackageYaml(dbQuerier pregistry.Query, packageName, downloadPath string) error {
408+
var errs []error
409+
410+
defaultChannel, err := dbQuerier.GetDefaultChannelForPackage(context.TODO(), packageName)
411+
if err != nil {
412+
return err
413+
}
414+
415+
channelList, err := dbQuerier.ListChannels(context.TODO(), packageName)
416+
if err != nil {
417+
return err
418+
}
419+
420+
channels := []pregistry.PackageChannel{}
421+
for _, ch := range channelList {
422+
csvName, err := dbQuerier.GetCurrentCSVNameForChannel(context.TODO(), packageName, ch)
423+
if err != nil {
424+
err = fmt.Errorf("error exporting bundle from image: %s", err)
425+
errs = append(errs, err)
426+
continue
427+
}
428+
channels = append(channels,
429+
pregistry.PackageChannel{
430+
Name: ch,
431+
CurrentCSVName: csvName,
432+
})
433+
}
434+
435+
manifest := pregistry.PackageManifest{
436+
PackageName: packageName,
437+
DefaultChannelName: defaultChannel,
438+
Channels: channels,
439+
}
440+
441+
manifestBytes, err := yaml.Marshal(&manifest)
442+
if err != nil {
443+
errs = append(errs, err)
444+
return utilerrors.NewAggregate(errs)
445+
}
446+
447+
err = bundle.WriteFile("package.yaml", downloadPath, manifestBytes)
448+
if err != nil {
449+
errs = append(errs, err)
450+
}
451+
452+
return utilerrors.NewAggregate(errs)
453+
}

‎pkg/lib/indexer/indexer_test.go

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package indexer
2+
3+
import (
4+
"database/sql"
5+
"io/ioutil"
6+
"os"
7+
"reflect"
8+
"sort"
9+
"testing"
10+
11+
"github.com/ghodss/yaml"
12+
"github.com/operator-framework/operator-registry/pkg/sqlite"
13+
14+
pregistry "github.com/operator-framework/operator-registry/pkg/registry"
15+
)
16+
17+
func TestGetBundlesToExport(t *testing.T) {
18+
expected := []string{"quay.io/olmtest/example-bundle:etcdoperator.v0.9.2", "quay.io/olmtest/example-bundle:etcdoperator.v0.9.0",
19+
"quay.io/olmtest/example-bundle:etcdoperator.v0.6.1"}
20+
sort.Strings(expected)
21+
22+
db, err := sql.Open("sqlite3", "./test/bundles.db")
23+
if err != nil {
24+
t.Fatalf("opening db: %s", err)
25+
}
26+
defer db.Close()
27+
28+
dbQuerier := sqlite.NewSQLLiteQuerierFromDb(db)
29+
if err != nil {
30+
t.Fatalf("creating querier: %s", err)
31+
}
32+
33+
bundleImages, err := getBundlesToExport(dbQuerier, "etcd")
34+
if err != nil {
35+
t.Fatalf("exporting bundles from db: %s", err)
36+
}
37+
sort.Strings(bundleImages)
38+
39+
if !reflect.DeepEqual(expected, bundleImages) {
40+
t.Fatalf("exporting images: expected matching bundlepaths: expected %s got %s", expected, bundleImages)
41+
}
42+
}
43+
44+
func TestGeneratePackageYaml(t *testing.T) {
45+
db, err := sql.Open("sqlite3", "./test/bundles.db")
46+
if err != nil {
47+
t.Fatalf("opening db: %s", err)
48+
}
49+
defer db.Close()
50+
51+
dbQuerier := sqlite.NewSQLLiteQuerierFromDb(db)
52+
if err != nil {
53+
t.Fatalf("creating querier: %s", err)
54+
}
55+
56+
err = generatePackageYaml(dbQuerier, "etcd", ".")
57+
if err != nil {
58+
t.Fatalf("writing package.yaml: %s", err)
59+
}
60+
61+
var expected pregistry.PackageManifest
62+
expectedBytes, _ := ioutil.ReadFile("./test/package.yaml")
63+
err = yaml.Unmarshal(expectedBytes, &expected)
64+
if err != nil {
65+
t.Fatalf("unmarshaling: %s", err)
66+
}
67+
68+
var actual pregistry.PackageManifest
69+
actualBytes, _ := ioutil.ReadFile("./package.yaml")
70+
err = yaml.Unmarshal(actualBytes, &actual)
71+
if err != nil {
72+
t.Fatalf("unmarshaling: %s", err)
73+
}
74+
75+
if !reflect.DeepEqual(expected, actual) {
76+
t.Fatalf("comparing package.yaml: expected #%v actual #%v", expected, actual)
77+
}
78+
79+
_ = os.RemoveAll("./package.yaml")
80+
}

‎pkg/lib/indexer/interfaces.go

+17
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,20 @@ func NewIndexDeleter(containerTool string, logger *logrus.Entry) IndexDeleter {
4747
Logger: logger,
4848
}
4949
}
50+
51+
//counterfeiter:generate . IndexExporter
52+
type IndexExporter interface {
53+
ExportFromIndex(ExportFromIndexRequest) error
54+
}
55+
56+
// NewIndexExporter is a constructor that returns an IndexExporter
57+
func NewIndexExporter(containerTool string, logger *logrus.Entry) IndexExporter {
58+
return ImageIndexer{
59+
DockerfileGenerator: containertools.NewDockerfileGenerator(containerTool, logger),
60+
CommandRunner: containertools.NewCommandRunner(containerTool, logger),
61+
LabelReader: containertools.NewLabelReader(containerTool, logger),
62+
ImageReader: containertools.NewImageReader(containerTool, logger),
63+
ContainerTool: containerTool,
64+
Logger: logger,
65+
}
66+
}

‎pkg/lib/indexer/test/bundles.db

1.36 MB
Binary file not shown.

‎pkg/lib/indexer/test/package.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
packageName: etcd
2+
channels:
3+
- name: alpha
4+
currentCSV: etcdoperator.v0.9.2
5+
- name: beta
6+
currentCSV: etcdoperator.v0.9.0
7+
- name: stable
8+
currentCSV: etcdoperator.v0.9.2
9+
defaultChannel: alpha

‎pkg/lib/registry/registry.go

+4-5
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,15 @@ import (
1010
"github.com/operator-framework/operator-registry/pkg/sqlite"
1111
)
1212

13-
1413
type RegistryUpdater struct {
1514
Logger *logrus.Entry
1615
}
1716

1817
type AddToRegistryRequest struct {
19-
Permissive bool
18+
Permissive bool
2019
InputDatabase string
2120
ContainerTool string
22-
Bundles []string
21+
Bundles []string
2322
}
2423

2524
func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
@@ -55,9 +54,9 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
5554
}
5655

5756
type DeleteFromRegistryRequest struct {
58-
Permissive bool
57+
Permissive bool
5958
InputDatabase string
60-
Packages []string
59+
Packages []string
6160
}
6261

6362
func (r RegistryUpdater) DeleteFromRegistry(request DeleteFromRegistryRequest) error {

‎pkg/registry/empty.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ func (EmptyQuery) GetBundleThatProvides(ctx context.Context, group, version, kin
5454

5555
func (EmptyQuery) ListImages(ctx context.Context) ([]string, error) {
5656
return nil, errors.New("empty querier: cannot get image list")
57-
5857
}
5958

6059
func (EmptyQuery) GetImagesForBundle(ctx context.Context, bundleName string) ([]string, error) {
@@ -65,6 +64,26 @@ func (EmptyQuery) GetApisForEntry(ctx context.Context, entryId int64) (provided
6564
return nil, nil, errors.New("empty querier: cannot apis")
6665
}
6766

67+
func (EmptyQuery) GetBundleVersion(ctx context.Context, image string) (string, error) {
68+
return "", errors.New("empty querier: cannot get version")
69+
}
70+
71+
func (EmptyQuery) GetBundlePathsForPackage(ctx context.Context, pkgName string) ([]string, error) {
72+
return nil, errors.New("empty querier: cannot get images")
73+
}
74+
75+
func (EmptyQuery) GetDefaultChannelForPackage(ctx context.Context, pkgName string) (string, error) {
76+
return "", errors.New("empty querier: cannot get default channel")
77+
}
78+
79+
func (EmptyQuery) ListChannels(ctx context.Context, pkgName string) ([]string, error) {
80+
return nil, errors.New("empty querier: cannot list channels")
81+
}
82+
83+
func (EmptyQuery) GetCurrentCSVNameForChannel(ctx context.Context, pkgName, channel string) (string, error) {
84+
return "", errors.New("empty querier: cannot get csv name for package and channel")
85+
}
86+
6887
var _ Query = &EmptyQuery{}
6988

7089
func NewEmptyQuerier() *EmptyQuery {

‎pkg/registry/interface.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,15 @@ type Query interface {
3535
// List all images for a particular bundle
3636
GetImagesForBundle(ctx context.Context, bundleName string) ([]string, error)
3737
// Get Provided and Required APIs for a particular bundle
38-
GetApisForEntry(ctx context.Context, entryId int64) (provided []*api.GroupVersionKind, required []*api.GroupVersionKind, err error)
38+
GetApisForEntry(ctx context.Context, entryID int64) (provided []*api.GroupVersionKind, required []*api.GroupVersionKind, err error)
39+
// Get Version of a Bundle Image
40+
GetBundleVersion(ctx context.Context, image string) (string, error)
41+
// List Images for Package
42+
GetBundlePathsForPackage(ctx context.Context, pkgName string) ([]string, error)
43+
// Get DefaultChannel for Package
44+
GetDefaultChannelForPackage(ctx context.Context, pkgName string) (string, error)
45+
// List channels for package
46+
ListChannels(ctx context.Context, pkgName string) ([]string, error)
47+
// Get CurrentCSV name for channel and package
48+
GetCurrentCSVNameForChannel(ctx context.Context, pkgName, channel string) (string, error)
3949
}

‎pkg/sqlite/image.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,17 @@ import (
1818

1919
// ImageLoader loads a bundle image of resources into the database
2020
type ImageLoader struct {
21-
store registry.Load
22-
image string
23-
directory string
21+
store registry.Load
22+
image string
23+
directory string
2424
containerTool string
2525
}
2626

2727
func NewSQLLoaderForImage(store registry.Load, image, containerTool string) *ImageLoader {
2828
return &ImageLoader{
29-
store: store,
30-
image: image,
31-
directory: "",
29+
store: store,
30+
image: image,
31+
directory: "",
3232
containerTool: containerTool,
3333
}
3434
}

‎pkg/sqlite/image_test.go

+168-17
Large diffs are not rendered by default.

‎pkg/sqlite/load.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ func (s *SQLLoader) AddPackageChannels(manifest registry.PackageManifest) error
270270

271271
// If we find 'replaces' in the circuit list then we've seen it already, break out
272272
if _, ok := replaceCycle[replaces]; ok {
273-
errs = append(errs, fmt.Errorf("Cycle detected, %s replaces %s", channelEntryCSVName, replaces))
273+
errs = append(errs, fmt.Errorf("Cycle detected, %s replaces %s", channelEntryCSVName, replaces))
274274
break
275275
}
276276
replaceCycle[replaces] = true
@@ -820,7 +820,7 @@ func (s *SQLLoader) updatePackageChannels(tx *sql.Tx, manifest registry.PackageM
820820
errs = append(errs, err)
821821
break
822822
}
823-
823+
824824
// check replaces
825825
replaces, err := channelEntryCSV.GetReplaces()
826826
if err != nil {

‎pkg/sqlite/query.go

+117-15
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func (s *SQLQuerier) GetBundle(ctx context.Context, pkgName, channelName, csvNam
124124
defer rows.Close()
125125

126126
if !rows.Next() {
127-
return nil, fmt.Errorf("no entry found for %s %s %s", pkgName, channelName, csvName)
127+
return nil, fmt.Errorf("no entry found for %s %s %s", pkgName, channelName, csvName)
128128
}
129129
var entryId sql.NullInt64
130130
var name sql.NullString
@@ -172,7 +172,7 @@ func (s *SQLQuerier) GetBundleForChannel(ctx context.Context, pkgName string, ch
172172
defer rows.Close()
173173

174174
if !rows.Next() {
175-
return nil, fmt.Errorf("no entry found for %s %s", pkgName, channelName)
175+
return nil, fmt.Errorf("no entry found for %s %s", pkgName, channelName)
176176
}
177177
var entryId sql.NullInt64
178178
var name sql.NullString
@@ -255,9 +255,8 @@ func (s *SQLQuerier) GetBundleThatReplaces(ctx context.Context, name, pkgName, c
255255
}
256256
defer rows.Close()
257257

258-
259258
if !rows.Next() {
260-
return nil, fmt.Errorf("no entry found for %s %s", pkgName, channelName)
259+
return nil, fmt.Errorf("no entry found for %s %s", pkgName, channelName)
261260
}
262261
var entryId sql.NullInt64
263262
var outName sql.NullString
@@ -473,14 +472,14 @@ func (s *SQLQuerier) GetImagesForBundle(ctx context.Context, csvName string) ([]
473472
return images, nil
474473
}
475474

476-
func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provided []*api.GroupVersionKind, required []*api.GroupVersionKind, err error) {
475+
func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryID int64) (provided []*api.GroupVersionKind, required []*api.GroupVersionKind, err error) {
477476
providedQuery := `SELECT DISTINCT api.group_name, api.version, api.kind, api.plural FROM api
478477
INNER JOIN api_provider ON (api.group_name=api_provider.group_name AND api.version=api_provider.version AND api.kind=api_provider.kind)
479478
WHERE api_provider.channel_entry_id=?`
480479

481-
providedRows, err := s.db.QueryContext(ctx, providedQuery, entryId)
480+
providedRows, err := s.db.QueryContext(ctx, providedQuery, entryID)
482481
if err != nil {
483-
return nil,nil, err
482+
return nil, nil, err
484483
}
485484

486485
provided = []*api.GroupVersionKind{}
@@ -497,10 +496,10 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provid
497496
return nil, nil, err
498497
}
499498
provided = append(provided, &api.GroupVersionKind{
500-
Group: groupName.String,
499+
Group: groupName.String,
501500
Version: versionName.String,
502-
Kind: kindName.String,
503-
Plural: pluralName.String,
501+
Kind: kindName.String,
502+
Plural: pluralName.String,
504503
})
505504
}
506505
if err := providedRows.Close(); err != nil {
@@ -511,9 +510,9 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provid
511510
INNER JOIN api_requirer ON (api.group_name=api_requirer.group_name AND api.version=api_requirer.version AND api.kind=api_requirer.kind)
512511
WHERE api_requirer.channel_entry_id=?`
513512

514-
requiredRows, err := s.db.QueryContext(ctx, requiredQuery, entryId)
513+
requiredRows, err := s.db.QueryContext(ctx, requiredQuery, entryID)
515514
if err != nil {
516-
return nil,nil, err
515+
return nil, nil, err
517516
}
518517
required = []*api.GroupVersionKind{}
519518
for requiredRows.Next() {
@@ -529,10 +528,10 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provid
529528
return nil, nil, err
530529
}
531530
required = append(required, &api.GroupVersionKind{
532-
Group: groupName.String,
531+
Group: groupName.String,
533532
Version: versionName.String,
534-
Kind: kindName.String,
535-
Plural: pluralName.String,
533+
Kind: kindName.String,
534+
Plural: pluralName.String,
536535
})
537536
}
538537
if err := requiredRows.Close(); err != nil {
@@ -541,3 +540,106 @@ func (s *SQLQuerier) GetApisForEntry(ctx context.Context, entryId int64) (provid
541540

542541
return
543542
}
543+
544+
func (s *SQLQuerier) GetBundleVersion(ctx context.Context, image string) (string, error) {
545+
query := `SELECT version FROM operatorbundle WHERE bundlepath=? LIMIT 1`
546+
rows, err := s.db.QueryContext(ctx, query, image)
547+
if err != nil {
548+
return "", err
549+
}
550+
defer rows.Close()
551+
552+
var version sql.NullString
553+
if rows.Next() {
554+
if err := rows.Scan(&version); err != nil {
555+
return "", err
556+
}
557+
}
558+
if version.Valid {
559+
return version.String, nil
560+
}
561+
return "", fmt.Errorf("bundle %s not found", image)
562+
}
563+
564+
func (s *SQLQuerier) GetBundlePathsForPackage(ctx context.Context, pkgName string) ([]string, error) {
565+
query := `SELECT DISTINCT bundlepath FROM operatorbundle
566+
INNER JOIN channel_entry ON operatorbundle.name=channel_entry.operatorbundle_name
567+
WHERE channel_entry.package_name=?`
568+
rows, err := s.db.QueryContext(ctx, query, pkgName)
569+
if err != nil {
570+
return nil, err
571+
}
572+
defer rows.Close()
573+
images := []string{}
574+
for rows.Next() {
575+
var imgName sql.NullString
576+
if err := rows.Scan(&imgName); err != nil {
577+
return nil, err
578+
}
579+
if imgName.Valid && imgName.String != "" {
580+
images = append(images, imgName.String)
581+
} else {
582+
return nil, fmt.Errorf("Index malformed: cannot find paths to bundle images")
583+
}
584+
}
585+
return images, nil
586+
}
587+
588+
func (s *SQLQuerier) GetDefaultChannelForPackage(ctx context.Context, pkgName string) (string, error) {
589+
query := `SELECT DISTINCT default_channel FROM package WHERE name=? LIMIT 1`
590+
rows, err := s.db.QueryContext(ctx, query, pkgName)
591+
if err != nil {
592+
return "", err
593+
}
594+
defer rows.Close()
595+
596+
var defaultChannel sql.NullString
597+
if rows.Next() {
598+
if err := rows.Scan(&defaultChannel); err != nil {
599+
return "", err
600+
}
601+
}
602+
if defaultChannel.Valid {
603+
return defaultChannel.String, nil
604+
}
605+
return "", nil
606+
}
607+
608+
func (s *SQLQuerier) ListChannels(ctx context.Context, pkgName string) ([]string, error) {
609+
query := `SELECT DISTINCT name FROM channel WHERE channel.package_name=?`
610+
rows, err := s.db.QueryContext(ctx, query, pkgName)
611+
if err != nil {
612+
return nil, err
613+
}
614+
defer rows.Close()
615+
channels := []string{}
616+
for rows.Next() {
617+
var chName sql.NullString
618+
if err := rows.Scan(&chName); err != nil {
619+
return nil, err
620+
}
621+
if chName.Valid {
622+
channels = append(channels, chName.String)
623+
}
624+
}
625+
return channels, nil
626+
}
627+
628+
func (s *SQLQuerier) GetCurrentCSVNameForChannel(ctx context.Context, pkgName, channel string) (string, error) {
629+
query := `SELECT DISTINCT head_operatorbundle_name FROM channel WHERE channel.package_name=? AND channel.name=?`
630+
rows, err := s.db.QueryContext(ctx, query, pkgName, channel)
631+
if err != nil {
632+
return "", err
633+
}
634+
defer rows.Close()
635+
var csvName sql.NullString
636+
if rows.Next() {
637+
if err := rows.Scan(&csvName); err != nil {
638+
return "", err
639+
}
640+
}
641+
if csvName.Valid {
642+
return csvName.String, nil
643+
}
644+
return "", nil
645+
}

0 commit comments

Comments
 (0)
Please sign in to comment.