Gangplank: allow for unbound pod creation in OpenShift or Podman

Instead of having a generic "ci" command, this supports generic
Kubernetes or Podman as "backend" workers for Gangplank.

The podman interface is primarly targeted for developer use-cases, while
the OpenShift kubernetes interface is for running Gangplank in arbitrary
clusters (i.e. GKE, AKE, etc.). This allows for CoreOS builds to happen
anywhere a pod-can be run without needing to know much more than some
YAML.

Signed-off-by: Ben Howard <[email protected]>

Author: Ben Howard

gangplank/cmd/build.go

@@ -45,6 +45,7 @@ func init() {
 // source or binary build strategies.
 func runOCP(c *cobra.Command, args []string) {
 	defer cancel()
+	defer ctx.Done()
 
 	// Terminal "keep alive" helper. When following logs via the `oc` commands,
 	// cloud-deployed will send an EOF. To get around the EOF, the func sends a

gangplank/cmd/ci.go

@@ -38,7 +38,7 @@ var (
 	cmdRoot = &cobra.Command{
 		Use:   "gangplank [command]",
 		Short: "COSA Gangplank",
-		Long: `Openshift COSA Job Runner
+		Long: `OpenShift COSA Job Runner
 Wrapper for COSA commands and templates`,
 		PersistentPreRun: preRun,
 	}

gangplank/cmd/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"os"
+
+	"github.com/coreos/gangplank/ocp"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+const cosaDefaultImage = "quay.io/coreos-assembler/coreos-assembler:latest"
+
+var (
+	cmdPod = &cobra.Command{
+		Use:   "pod",
+		Short: "Execute COSA command in an OpenShift Cluster (default) or Podman",
+		Run:   runPod,
+	}
+
+	// cosaOverrideImage uses a different image
+	cosaOverrideImage string
+
+	// serviceAccount is the service acount to use for pod creation
+	// and reading of the secrets.
+	serviceAccount string
+
+	// cosaCmds is used to define the commands to run
+	cosaCmds []string
+
+	// Run CI pod via podman (out of cluster)
+	cosaViaPodman bool
+
+	// workDir is used for podman mode
+	cosaWorkDir string
+)
+
+func init() {
+	cmdRoot.AddCommand(cmdPod)
+	cmdPod.Flags().BoolVarP(&cosaViaPodman, "podman", "", false, "use podman to execute task")
+	cmdPod.Flags().StringSliceVarP(&cosaCmds, "cmd", "c", []string{}, "commands to run")
+	cmdPod.Flags().StringVarP(&cosaOverrideImage, "image", "i", "", "use an alternative image")
+	cmdPod.Flags().StringVarP(&cosaWorkDir, "workDir", "w", "", "podman mode - workdir to use")
+	cmdPod.Flags().StringVarP(&serviceAccount, "serviceaccount", "a", "", "service account to use")
+}
+
+// runPod is the Jenkins/CI interface into Gangplank. It "mocks"
+// the OpenShift buildconfig API with just-enough information to be
+// useful.
+func runPod(c *cobra.Command, args []string) {
+	defer cancel()
+
+	inCluster := true
+	launchMode := "OpenShift"
+	if cosaViaPodman {
+		inCluster = false
+		launchMode = "Podman"
+		if cosaOverrideImage == "" {
+			cosaOverrideImage = cosaDefaultImage
+		}
+		if cosaWorkDir == "" {
+			cosaWorkDir, _ = os.Getwd()
+		}
+	}
+	pb, err := ocp.NewPodBuilder(ctx, inCluster, cosaOverrideImage, serviceAccount, specFile, cosaWorkDir)
+	if err != nil {
+		log.Fatalf("failed to define builder pod: %v", err)
+	}
+
+	log.Infof("Lauching %s worker pod(s)", launchMode)
+	if err := pb.Exec(ctx, cosaWorkDir); err != nil {
+		log.Fatalf("failed to execute CI builder: %v", err)
+	}
+}

gangplank/cmd/pod.go

@@ -23,6 +23,7 @@ import (
 	"github.com/coreos/gangplank/spec"
 	buildapiv1 "github.com/openshift/api/build/v1"
 	log "github.com/sirupsen/logrus"
+	"k8s.io/client-go/kubernetes"
 )
 
 var (
@@ -56,6 +57,9 @@ type buildConfig struct {
 
 	// Internal copy of the JobSpec
 	JobSpec spec.JobSpec
+
+	KubeClient  *kubernetes.Clientset
+	KubeProject string
 }
 
 // newBC accepts a context and returns a buildConfig
@@ -74,31 +78,30 @@ func newBC() (*buildConfig, error) {
 		}
 	}
 
-	// Open the Kubernetes Client
-	ac, pn, kubeErr := k8sInClusterClient()
-
 	// Init the OpenShift Build API Client.
-	buildAPIErr := ocpBuildClient()
-	if buildAPIErr != nil && buildAPIErr != ErrNoOCPBuildSpec {
-		log.Errorf("Failed to initalized the OpenShift Build API Client: %v", buildAPIErr)
-		return nil, buildAPIErr
-	}
-
-	if kubeErr != nil && buildAPIErr != nil {
-		return nil, ErrInvalidOCPMode
+	if err := ocpBuildClient(); err != nil {
+		log.WithError(err).Error("Failed to initalized the OpenShift Build API Client")
+		return nil, err
 	}
 
 	// Query Kubernetes to find out what this pods network identity is.
 	// TODO: remove this CI exception once we have a kubernetes mock
 	if !forceNotInCluster {
+		ac, pn, kubeErr := k8sInClusterClient()
+		if kubeErr != nil {
+			return nil, ErrInvalidOCPMode
+		}
+		v.KubeClient = ac
+		v.KubeProject = pn
+
 		v.HostPod = fmt.Sprintf("%s-%s-build",
 			apiBuild.Annotations[buildapiv1.BuildConfigAnnotation],
 			apiBuild.Annotations[buildapiv1.BuildNumberAnnotation],
 		)
 
-		_, ok := apiBuild.Annotations[ciRunnerTag]
+		_, ok := apiBuild.Annotations[podBuildRunnerTag]
 		if ok {
-			v.HostIP = apiBuild.Annotations[fmt.Sprintf(ciAnnotation, "IP")]
+			v.HostIP = apiBuild.Annotations[fmt.Sprintf(podBuildAnnotation, "IP")]
 		} else {
 			log.Info("Querying for pod ID")
 			hIP, err := getPodIP(ac, pn, v.HostPod)
@@ -140,6 +143,8 @@ func newBC() (*buildConfig, error) {
 			v.JobSpec = njs
 		}
 	}
+
+	log.Info("Running Pod in buildconfig mode.")
 	return &v, nil
 }
 
@@ -152,11 +157,6 @@ func (bc *buildConfig) Exec(ctx context.Context) error {
 		return err
 	}
 
-	ac, pn, err := k8sInClusterClient()
-	if err != nil {
-		return fmt.Errorf("failed create a kubernetes client: %w", err)
-	}
-
 	// Define, but do not start minio.
 	m := newMinioServer()
 	m.dir = cosaSrvDir
@@ -200,6 +200,7 @@ binary build interface.`)
 	if err := m.start(ctx); err != nil {
 		return fmt.Errorf("failed to start Minio: %w", err)
 	}
+	defer m.kill()
 
 	if err := m.ensureBucketExists(ctx, "builds"); err != nil {
 		return err
@@ -296,12 +297,12 @@ binary build interface.`)
 		}
 
 		index := n + 1
-		cpod, err := NewCosaPodder(ctx, apiBuild, ac, pn, index)
+		cpod, err := NewCosaPodder(ctx, apiBuild, bc.KubeClient, bc.KubeProject, index)
 		if err != nil {
 			log.Errorf("FAILED TO CREATE POD DEFINITION: %v", err)
 			continue
 		}
-		if err := cpod.WorkerRunner(eVars); err != nil {
+		if err := cpod.WorkerRunner(ctx, eVars); err != nil {
 			log.Errorf("FAILED stage: %v", err)
 		}
 	}

gangplank/ocp/bc.go

@@ -18,7 +18,7 @@ func init() {
 }
 
 func TestNoEnv(t *testing.T) {
-	if _, err := newBC(); err != ErrInvalidOCPMode {
+	if _, err := newBC(); err != ErrNoOCPBuildSpec {
 		t.Errorf("failed to raise error\n   want: %v\n    got: %v", ErrInvalidOCPMode, err)
 	}
 }

gangplank/ocp/bc_ci_test.go

@@ -1,6 +1,11 @@
 package ocp
 
-import "context"
+import (
+	"context"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
 
 // Builder implements the Build
 type Builder interface {
@@ -13,6 +18,13 @@ var (
 	cosaSrvDir = defaultContextDir
 )
 
+func init() {
+	if _, ok := os.LookupEnv(localPodEnvVar); ok {
+		log.Infof("EnvVar %s defined, using local pod mode", localPodEnvVar)
+		forceNotInCluster = true
+	}
+}
+
 // NewBuilder returns a Builder. NewBuilder determines what
 // "Builder" to return.
 func NewBuilder(ctx context.Context) (Builder, error) {