//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 builder.go ImageAppender package appregistry import ( "archive/tar" "compress/gzip" "context" "database/sql" "fmt" "io" "io/ioutil" "os" "path" "path/filepath" "k8s.io/klog" "github.com/operator-framework/operator-registry/pkg/apprclient" "github.com/operator-framework/operator-registry/pkg/sqlite" ) type RegistryImageBuilder interface { Build() error } type ImageAppender interface { Append(from, to, layer string) error } type ImageAppendFunc func(from, to, layer string) error func (f ImageAppendFunc) Append(from, to, layer string) error { return f(from, to, layer) } type AppregistryImageBuilder struct { Appender ImageAppender // options From, To string AuthToken string AppRegistryEndpoint string AppRegistryOrg string DatabasePath string CacheDir string // derived CleanOutput bool ManifestDir string DatabaseDir string client apprclient.Client } func NewAppregistryImageBuilder(options ...AppregistryBuildOption) (*AppregistryImageBuilder, error) { config := DefaultAppregistryBuildOptions() config.Apply(options) if err := config.Complete(); err != nil { return nil, err } if err := config.Validate(); err != nil { return nil, err } return &AppregistryImageBuilder{ Appender: config.Appender, From: config.From, To: config.To, AuthToken: config.AuthToken, AppRegistryEndpoint: config.AppRegistryEndpoint, AppRegistryOrg: config.AppRegistryOrg, DatabasePath: config.DatabasePath, CacheDir: config.CacheDir, CleanOutput: config.CleanOutput, ManifestDir: config.ManifestDir, DatabaseDir: config.DatabaseDir, client: config.Client, }, nil } func (b *AppregistryImageBuilder) Build() error { defer func() { if !b.CleanOutput { return } if err := os.RemoveAll(b.CacheDir); err != nil { klog.Warningf("unable to clean %s", b.CacheDir) } }() downloader := NewManifestDownloader(b.client) if err := downloader.DownloadManifests(b.ManifestDir, b.AppRegistryOrg); err != nil { return err } if !hasManifests(b.ManifestDir) { return fmt.Errorf("no manifests downloaded from appregistry %s/%s", b.AppRegistryEndpoint, b.AppRegistryOrg) } klog.V(4).Infof("downloaded manifests to %s\n", b.ManifestDir) if err := BuildDatabase(b.ManifestDir, b.DatabasePath); err != nil { return err } klog.V(4).Infof("database written %s\n", b.DatabasePath) if b.To == "" { return nil } archivePath, err := BuildLayer(b.DatabaseDir) if err != nil { return err } klog.V(4).Infof("built db layer %s\n", archivePath) return b.Appender.Append(b.From, b.To, archivePath) } func BuildDatabase(manifestPath, databasePath string) error { db, err := sql.Open("sqlite3", databasePath) if err != nil { return err } dbLoader, err := sqlite.NewSQLLiteLoader(db) if err != nil { return err } defer func() { if err := db.Close(); err != nil { klog.Warningf(err.Error()) } }() if err := dbLoader.Migrate(context.TODO()); err != nil { return err } loader := sqlite.NewSQLLoaderForDirectory(dbLoader, manifestPath) if err := loader.Populate(); err != nil { klog.Warningf("error building database: %s", err.Error()) } return nil } func BuildLayer(directory string) (string, error) { archiveDir, err := ioutil.TempDir("", "archive-") if err != nil { return "", err } archive, err := os.Create(path.Join(archiveDir, "layer.tar.gz")) if err != nil { return "", err } defer func() { if err := archive.Close(); err != nil { klog.Warningf("error closing file: %s", err.Error()) } }() gzipWriter := gzip.NewWriter(archive) defer func() { if err := gzipWriter.Close(); err != nil { klog.Warningf("error closing writer: %s", err.Error()) } }() writer := tar.NewWriter(gzipWriter) defer func() { if err := writer.Close(); err != nil { klog.Warningf("error closing writer: %s", err.Error()) } }() if err := filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.Mode().IsRegular() { return nil } header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { return err } err = writer.WriteHeader(header) if err != nil { return err } // if it's a directory, just write the header and continue if info.IsDir() { return nil } file, err := os.Open(path) if err != nil { return err } defer func() { if err := file.Close(); err != nil { klog.Warningf("error closing file: %s", err.Error()) } }() _, err = io.Copy(writer, file) if err != nil { return err } return err }); err != nil { return "", err } return archive.Name(), nil } func hasManifests(path string) bool { files, err := ioutil.ReadDir(path) if err != nil { return false } return len(files) > 0 }