From dfa69ee73176f950fd66656a52914f0212566f89 Mon Sep 17 00:00:00 2001 From: Pete Savage Date: Fri, 10 Jan 2025 14:14:00 +0000 Subject: [PATCH 1/6] Stateful Set test --- .../v1alpha1/clowdapp_types.go | 6 ++ .../clowdapp_reconciliation.go | 1 + .../providers/autoscaler/keda.go | 13 ++-- .../providers/deployment/default.go | 3 + .../providers/deployment/provider.go | 64 +++++++++++++++++++ .../providers/serviceaccount/default.go | 8 +-- 6 files changed, 85 insertions(+), 10 deletions(-) diff --git a/apis/cloud.redhat.com/v1alpha1/clowdapp_types.go b/apis/cloud.redhat.com/v1alpha1/clowdapp_types.go index e2aad7be7..062a3391e 100644 --- a/apis/cloud.redhat.com/v1alpha1/clowdapp_types.go +++ b/apis/cloud.redhat.com/v1alpha1/clowdapp_types.go @@ -238,6 +238,12 @@ type Deployment struct { DeploymentStrategy *DeploymentStrategy `json:"deploymentStrategy,omitempty"` Metadata DeploymentMetadata `json:"metadata,omitempty"` + + Stateful StatefulSpec `json:"statefulSpec,omitempty"` +} + +type StatefulSpec struct { + Enabled bool `json:"enabled,omitempty"` } func (d *Deployment) GetReplicaCount() *int32 { diff --git a/controllers/cloud.redhat.com/clowdapp_reconciliation.go b/controllers/cloud.redhat.com/clowdapp_reconciliation.go index 99c11752d..7967d99f8 100644 --- a/controllers/cloud.redhat.com/clowdapp_reconciliation.go +++ b/controllers/cloud.redhat.com/clowdapp_reconciliation.go @@ -288,6 +288,7 @@ var applyOrder = []string{ "Service", "Secret", "Deployment", + "StatefulSet", "Job", "CronJob", "ScaledObject", diff --git a/controllers/cloud.redhat.com/providers/autoscaler/keda.go b/controllers/cloud.redhat.com/providers/autoscaler/keda.go index 0c8bc8dab..bdc390d9e 100644 --- a/controllers/cloud.redhat.com/providers/autoscaler/keda.go +++ b/controllers/cloud.redhat.com/providers/autoscaler/keda.go @@ -8,8 +8,8 @@ import ( "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers" deployProvider "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/deployment" keda "github.com/kedacore/keda/v2/apis/keda/v1alpha1" - apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) func makeAutoScalers(deployment *crd.Deployment, app *crd.ClowdApp, c *config.AppConfig, asp *providers.Provider) error { @@ -19,12 +19,13 @@ func makeAutoScalers(deployment *crd.Deployment, app *crd.ClowdApp, c *config.Ap return err } - d := &apps.Deployment{} - if err := asp.Cache.Get(deployProvider.CoreDeployment, d, nn); err != nil { + obj, err := deployProvider.GetClientObject(deployment, asp.Cache, nn) + + if err != nil { return err } - initAutoScaler(asp.Env, app, d, s, nn, deployment, c) + initAutoScaler(asp.Env, app, obj, s, nn, deployment, c) return asp.Cache.Update(CoreAutoScaler, s) } @@ -34,14 +35,14 @@ func ProvideKedaAutoScaler(app *crd.ClowdApp, c *config.AppConfig, asp *provider return err } -func initAutoScaler(env *crd.ClowdEnvironment, app *crd.ClowdApp, d *apps.Deployment, s *keda.ScaledObject, nn types.NamespacedName, deployment *crd.Deployment, c *config.AppConfig) { +func initAutoScaler(env *crd.ClowdEnvironment, app *crd.ClowdApp, obj client.Object, s *keda.ScaledObject, nn types.NamespacedName, deployment *crd.Deployment, c *config.AppConfig) { labels := app.GetLabels() labels["pod"] = nn.Name app.SetObjectMeta(s, crd.Name(nn.Name), crd.Labels(labels)) // Set up the watcher to watch the Deployment we created earlier. scalerSpec := keda.ScaledObjectSpec{ - ScaleTargetRef: &keda.ScaleTarget{Name: d.Name, Kind: d.Kind, APIVersion: d.APIVersion}, + ScaleTargetRef: &keda.ScaleTarget{Name: obj.GetName(), Kind: obj.GetObjectKind().GroupVersionKind().Kind, APIVersion: obj.GetObjectKind().GroupVersionKind().Version}, PollingInterval: deployment.AutoScaler.PollingInterval, CooldownPeriod: deployment.AutoScaler.CooldownPeriod, Advanced: deployment.AutoScaler.Advanced, diff --git a/controllers/cloud.redhat.com/providers/deployment/default.go b/controllers/cloud.redhat.com/providers/deployment/default.go index 40d7c3571..bd98dd064 100644 --- a/controllers/cloud.redhat.com/providers/deployment/default.go +++ b/controllers/cloud.redhat.com/providers/deployment/default.go @@ -14,8 +14,11 @@ type deploymentProvider struct { // CoreDeployment is the deployment for the apps deployments. var CoreDeployment = rc.NewMultiResourceIdent(ProvName, "core_deployment", &apps.Deployment{}) +var CoreStatefulSet = rc.NewMultiResourceIdent(ProvName, "core_statefulset", &apps.StatefulSet{}) + func NewDeploymentProvider(p *providers.Provider) (providers.ClowderProvider, error) { p.Cache.AddPossibleGVKFromIdent(CoreDeployment) + p.Cache.AddPossibleGVKFromIdent(CoreStatefulSet) return &deploymentProvider{Provider: *p}, nil } diff --git a/controllers/cloud.redhat.com/providers/deployment/provider.go b/controllers/cloud.redhat.com/providers/deployment/provider.go index 67aab5723..95c7d8b95 100644 --- a/controllers/cloud.redhat.com/providers/deployment/provider.go +++ b/controllers/cloud.redhat.com/providers/deployment/provider.go @@ -1,7 +1,15 @@ package deployment import ( + "fmt" + + crd "github.com/RedHatInsights/clowder/apis/cloud.redhat.com/v1alpha1" "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers" + rc "github.com/RedHatInsights/rhc-osdk-utils/resourceCache" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) // ProvName sets the provider name identifier @@ -15,3 +23,59 @@ func GetDeployment(c *providers.Provider) (providers.ClowderProvider, error) { func init() { providers.ProvidersRegistration.Register(GetDeployment, 0, ProvName) } + +func GetPodTemplateFromObject(deployment *crd.Deployment, rc *rc.ObjectCache, nn types.NamespacedName) (*v1.PodTemplateSpec, error) { + obj, err := GetClientObject(deployment, rc, nn) + if err != nil { + return nil, err + } + + switch v := obj.(type) { + case *apps.StatefulSet: + return &v.Spec.Template, nil + case *apps.Deployment: + return &v.Spec.Template, nil + } + return nil, fmt.Errorf("no valid type") +} + +func GetClientObject(deployment *crd.Deployment, rc *rc.ObjectCache, nn types.NamespacedName) (client.Object, error) { + if deployment.Stateful.Enabled { + ss := &apps.StatefulSet{} + if err := rc.Get(CoreStatefulSet, ss, nn); err != nil { + return nil, err + } + return ss, nil + } + + d := &apps.Deployment{} + if err := rc.Get(CoreDeployment, d, nn); err != nil { + return nil, err + } + return d, nil +} + +func UpdatePodTemplate(deployment *crd.Deployment, podTemplate *v1.PodTemplateSpec, rc *rc.ObjectCache, nn types.NamespacedName) error { + + if deployment.Stateful.Enabled { + ss := &apps.Deployment{} + if err := rc.Get(CoreStatefulSet, ss, nn); err != nil { + return err + } + ss.Spec.Template = *podTemplate + if err := rc.Update(CoreStatefulSet, ss); err != nil { + return err + } + } + + d := &apps.Deployment{} + if err := rc.Get(CoreDeployment, d, nn); err != nil { + return err + } + d.Spec.Template = *podTemplate + if err := rc.Update(CoreDeployment, d); err != nil { + return err + } + + return nil +} diff --git a/controllers/cloud.redhat.com/providers/serviceaccount/default.go b/controllers/cloud.redhat.com/providers/serviceaccount/default.go index be67433fe..8c0c7a6c1 100644 --- a/controllers/cloud.redhat.com/providers/serviceaccount/default.go +++ b/controllers/cloud.redhat.com/providers/serviceaccount/default.go @@ -114,11 +114,11 @@ func (sa *serviceaccountProvider) Provide(app *crd.ClowdApp) error { } for _, dep := range app.Spec.Deployments { - d := &apps.Deployment{} innerDeployment := dep nn := app.GetDeploymentNamespacedName(&innerDeployment) - if err := sa.Cache.Get(deployment.CoreDeployment, d, nn); err != nil { + podTemplate, err := deployment.GetPodTemplateFromObject(&innerDeployment, sa.Cache, nn) + if err != nil { return err } @@ -128,8 +128,8 @@ func (sa *serviceaccountProvider) Provide(app *crd.ClowdApp) error { return err } - d.Spec.Template.Spec.ServiceAccountName = nn.Name - if err := sa.Cache.Update(deployment.CoreDeployment, d); err != nil { + podTemplate.Spec.ServiceAccountName = nn.Name + if err := deployment.UpdatePodTemplate(&innerDeployment, podTemplate, sa.Cache, nn); err != nil { return err } From e089e9365fc2bdfa1d9b2a6f7d341246a8924f55 Mon Sep 17 00:00:00 2001 From: Pete Savage Date: Fri, 10 Jan 2025 18:33:06 +0000 Subject: [PATCH 2/6] Next round of changes --- .../providers/autoscaler/simple.go | 55 +++---- .../providers/confighash/config.go | 34 +++-- .../providers/confighash/default.go | 15 ++ .../providers/deployment/default.go | 12 +- .../providers/deployment/impl.go | 136 ++++++++++++++---- 5 files changed, 186 insertions(+), 66 deletions(-) diff --git a/controllers/cloud.redhat.com/providers/autoscaler/simple.go b/controllers/cloud.redhat.com/providers/autoscaler/simple.go index f86fab5c1..31e82af85 100644 --- a/controllers/cloud.redhat.com/providers/autoscaler/simple.go +++ b/controllers/cloud.redhat.com/providers/autoscaler/simple.go @@ -4,15 +4,15 @@ import ( "fmt" res "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client" crd "github.com/RedHatInsights/clowder/apis/cloud.redhat.com/v1alpha1" "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/config" "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/errors" "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers" deployProvider "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/deployment" - apps "k8s.io/api/apps/v1" v2 "k8s.io/api/autoscaling/v2" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -25,11 +25,11 @@ const ( // Creates a simple HPA in the resource cache for the deployment and ClowdApp func ProvideSimpleAutoScaler(app *crd.ClowdApp, appConfig *config.AppConfig, sp *providers.Provider, deployment crd.Deployment) error { - cachedDeployment, err := getDeploymentFromCache(&deployment, app, sp) + coreObject, err := getcoreObjectFromCache(&deployment, app, sp) if err != nil { return errors.Wrap("Could not get deployment from resource cache", err) } - hpaMaker := newSimpleHPAMaker(&deployment, app, appConfig, cachedDeployment) + hpaMaker := newSimpleHPAMaker(&deployment, app, appConfig, coreObject) hpaResource := hpaMaker.getResource() err = cacheAutoscaler(app, sp, deployment, hpaResource) @@ -47,32 +47,33 @@ func cacheAutoscaler(app *crd.ClowdApp, sp *providers.Provider, deployment crd.D } // Get the core apps.Deployment from the provider cache -func getDeploymentFromCache(clowdDeployment *crd.Deployment, app *crd.ClowdApp, sp *providers.Provider) (*apps.Deployment, error) { +func getcoreObjectFromCache(clowdDeployment *crd.Deployment, app *crd.ClowdApp, sp *providers.Provider) (client.Object, error) { nn := app.GetDeploymentNamespacedName(clowdDeployment) - d := &apps.Deployment{} - if err := sp.Cache.Get(deployProvider.CoreDeployment, d, nn); err != nil { - return d, err + + obj, err := deployProvider.GetClientObject(clowdDeployment, sp.Cache, nn) + if err != nil { + return nil, err } - return d, nil + return obj, nil } // Factory for the simpleHPAMaker -func newSimpleHPAMaker(deployment *crd.Deployment, app *crd.ClowdApp, appConfig *config.AppConfig, coreDeployment *apps.Deployment) simpleHPAMaker { +func newSimpleHPAMaker(deployment *crd.Deployment, app *crd.ClowdApp, appConfig *config.AppConfig, coreObject client.Object) simpleHPAMaker { return simpleHPAMaker{ - deployment: deployment, - app: app, - appConfig: appConfig, - coreDeployment: coreDeployment, + deployment: deployment, + app: app, + appConfig: appConfig, + coreObject: coreObject, } } // Creates a simple HPA and stores references // to the resources and dependencies it requires type simpleHPAMaker struct { - deployment *crd.Deployment - app *crd.ClowdApp - appConfig *config.AppConfig - coreDeployment *apps.Deployment + deployment *crd.Deployment + app *crd.ClowdApp + appConfig *config.AppConfig + coreObject client.Object } // Constructs the HPA in 2 parts: the HPA itself and the metric spec @@ -96,13 +97,13 @@ func (d *simpleHPAMaker) makeHPA() v2.HorizontalPodAutoscaler { UID: d.app.UID, }}, Name: name, - Namespace: d.coreDeployment.Namespace, + Namespace: d.coreObject.GetNamespace(), }, Spec: v2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: v2.CrossVersionObjectReference{ APIVersion: DeploymentAPIVersion, Kind: DeploymentKind, - Name: d.coreDeployment.Name, + Name: d.coreObject.GetName(), }, MinReplicas: &d.deployment.AutoScalerSimple.Replicas.Min, MaxReplicas: d.deployment.AutoScalerSimple.Replicas.Max, @@ -116,43 +117,43 @@ func (d *simpleHPAMaker) makeMetricsSpecs() []v2.MetricSpec { metricsSpecs := []v2.MetricSpec{} if d.deployment.AutoScalerSimple.RAM.ScaleAtUtilization != 0 { - metricsSpec := d.makeAverageUtilizationMetricSpec(v1.ResourceMemory, d.deployment.AutoScalerSimple.RAM.ScaleAtUtilization) + metricsSpec := d.makeAverageUtilizationMetricSpec(core.ResourceMemory, d.deployment.AutoScalerSimple.RAM.ScaleAtUtilization) metricsSpecs = append(metricsSpecs, metricsSpec) } if d.deployment.AutoScalerSimple.RAM.ScaleAtValue != "" { threshold := res.MustParse(d.deployment.AutoScalerSimple.RAM.ScaleAtValue) - metricsSpec := d.makeAverageValueMetricSpec(v1.ResourceMemory, threshold) + metricsSpec := d.makeAverageValueMetricSpec(core.ResourceMemory, threshold) metricsSpecs = append(metricsSpecs, metricsSpec) } if d.deployment.AutoScalerSimple.CPU.ScaleAtUtilization != 0 { - metricsSpec := d.makeAverageUtilizationMetricSpec(v1.ResourceCPU, d.deployment.AutoScalerSimple.CPU.ScaleAtUtilization) + metricsSpec := d.makeAverageUtilizationMetricSpec(core.ResourceCPU, d.deployment.AutoScalerSimple.CPU.ScaleAtUtilization) metricsSpecs = append(metricsSpecs, metricsSpec) } if d.deployment.AutoScalerSimple.CPU.ScaleAtValue != "" { threshold := res.MustParse(d.deployment.AutoScalerSimple.CPU.ScaleAtValue) - metricsSpec := d.makeAverageValueMetricSpec(v1.ResourceCPU, threshold) + metricsSpec := d.makeAverageValueMetricSpec(core.ResourceCPU, threshold) metricsSpecs = append(metricsSpecs, metricsSpec) } return metricsSpecs } -func (d *simpleHPAMaker) makeAverageValueMetricSpec(resource v1.ResourceName, threshold res.Quantity) v2.MetricSpec { +func (d *simpleHPAMaker) makeAverageValueMetricSpec(resource core.ResourceName, threshold res.Quantity) v2.MetricSpec { ms := d.makeBasicMetricSpec(resource) ms.Resource.Target.Type = v2.AverageValueMetricType ms.Resource.Target.AverageValue = &threshold return ms } -func (d *simpleHPAMaker) makeAverageUtilizationMetricSpec(resource v1.ResourceName, threshold int32) v2.MetricSpec { +func (d *simpleHPAMaker) makeAverageUtilizationMetricSpec(resource core.ResourceName, threshold int32) v2.MetricSpec { ms := d.makeBasicMetricSpec(resource) ms.Resource.Target.Type = v2.UtilizationMetricType ms.Resource.Target.AverageUtilization = &threshold return ms } -func (d *simpleHPAMaker) makeBasicMetricSpec(resource v1.ResourceName) v2.MetricSpec { +func (d *simpleHPAMaker) makeBasicMetricSpec(resource core.ResourceName) v2.MetricSpec { ms := v2.MetricSpec{ Type: v2.MetricSourceType("Resource"), Resource: &v2.ResourceMetricSource{ diff --git a/controllers/cloud.redhat.com/providers/confighash/config.go b/controllers/cloud.redhat.com/providers/confighash/config.go index d7769c8fc..b018f4b94 100644 --- a/controllers/cloud.redhat.com/providers/confighash/config.go +++ b/controllers/cloud.redhat.com/providers/confighash/config.go @@ -105,8 +105,8 @@ func (ch *confighashProvider) volSecret(app *crd.ClowdApp, volume core.Volume) e return ch.HashCache.AddClowdObjectToObject(app, sec) } -func (ch *confighashProvider) iterateEnvVars(app *crd.ClowdApp, deployment apps.Deployment) error { - for _, cont := range deployment.Spec.Template.Spec.Containers { +func (ch *confighashProvider) iterateEnvVars(app *crd.ClowdApp, podTemplate core.PodTemplateSpec) error { + for _, cont := range podTemplate.Spec.Containers { for _, env := range cont.Env { if err := ch.envConfigMap(app, env); err != nil { return err @@ -120,8 +120,8 @@ func (ch *confighashProvider) iterateEnvVars(app *crd.ClowdApp, deployment apps. return nil } -func (ch *confighashProvider) iterateVolumes(app *crd.ClowdApp, deployment apps.Deployment) error { - for _, volume := range deployment.Spec.Template.Spec.Volumes { +func (ch *confighashProvider) iterateVolumes(app *crd.ClowdApp, podTemplate core.PodTemplateSpec) error { + for _, volume := range podTemplate.Spec.Volumes { if err := ch.volConfigMap(app, volume); err != nil { return err } @@ -133,13 +133,13 @@ func (ch *confighashProvider) iterateVolumes(app *crd.ClowdApp, deployment apps. return nil } -func (ch *confighashProvider) updateHashCache(dList *apps.DeploymentList, app *crd.ClowdApp) error { - for _, deployment := range dList.Items { - deploy := deployment - if err := ch.iterateEnvVars(app, deploy); err != nil { +func (ch *confighashProvider) updateHashCache(podTemplateSpecList *[]core.PodTemplateSpec, app *crd.ClowdApp) error { + for _, podTemplate := range *podTemplateSpecList { + pt := podTemplate + if err := ch.iterateEnvVars(app, pt); err != nil { return err } - if err := ch.iterateVolumes(app, deploy); err != nil { + if err := ch.iterateVolumes(app, pt); err != nil { return err } } @@ -161,7 +161,21 @@ func (ch *confighashProvider) persistConfig(app *crd.ClowdApp) (string, error) { return "", err } - if err := ch.updateHashCache(&dList, app); err != nil { + ssList := apps.StatefulSetList{} + if err := ch.Cache.List(deployProvider.CoreStatefulSet, &ssList); err != nil { + return "", err + } + + podTemplateList := []core.PodTemplateSpec{} + for _, item := range dList.Items { + podTemplateList = append(podTemplateList, item.Spec.Template) + } + + for _, item := range ssList.Items { + podTemplateList = append(podTemplateList, item.Spec.Template) + } + + if err := ch.updateHashCache(&podTemplateList, app); err != nil { return "", err } diff --git a/controllers/cloud.redhat.com/providers/confighash/default.go b/controllers/cloud.redhat.com/providers/confighash/default.go index 6b4c5bd78..1a4717156 100644 --- a/controllers/cloud.redhat.com/providers/confighash/default.go +++ b/controllers/cloud.redhat.com/providers/confighash/default.go @@ -53,6 +53,21 @@ func (ch *confighashProvider) Provide(app *crd.ClowdApp) error { } } + ssList := apps.StatefulSetList{} + if err := ch.Cache.List(deployProvider.CoreStatefulSet, &ssList); err != nil { + return err + } + + for _, statefulset := range ssList.Items { + ssInner := statefulset + annotations := map[string]string{"configHash": hash} + utils.UpdateAnnotations(&ssInner.Spec.Template, annotations) + + if err := ch.Cache.Update(deployProvider.CoreStatefulSet, &ssInner); err != nil { + return err + } + } + jList := batch.CronJobList{} if err := ch.Cache.List(cronjobProvider.CoreCronJob, &jList); err != nil { return err diff --git a/controllers/cloud.redhat.com/providers/deployment/default.go b/controllers/cloud.redhat.com/providers/deployment/default.go index bd98dd064..22960cf20 100644 --- a/controllers/cloud.redhat.com/providers/deployment/default.go +++ b/controllers/cloud.redhat.com/providers/deployment/default.go @@ -14,6 +14,7 @@ type deploymentProvider struct { // CoreDeployment is the deployment for the apps deployments. var CoreDeployment = rc.NewMultiResourceIdent(ProvName, "core_deployment", &apps.Deployment{}) +// CoreStatefulSet is the statefulSet for the apps deployments. var CoreStatefulSet = rc.NewMultiResourceIdent(ProvName, "core_statefulset", &apps.StatefulSet{}) func NewDeploymentProvider(p *providers.Provider) (providers.ClowderProvider, error) { @@ -29,9 +30,14 @@ func (dp *deploymentProvider) EnvProvide() error { func (dp *deploymentProvider) Provide(app *crd.ClowdApp) error { for _, deployment := range app.Spec.Deployments { - - if err := dp.makeDeployment(deployment, app); err != nil { - return err + if deployment.Stateful.Enabled { + if err := dp.makeStatefulSet(deployment, app); err != nil { + return err + } + } else { + if err := dp.makeDeployment(deployment, app); err != nil { + return err + } } } return nil diff --git a/controllers/cloud.redhat.com/providers/deployment/impl.go b/controllers/cloud.redhat.com/providers/deployment/impl.go index 7dc82cdc0..25b8067d6 100644 --- a/controllers/cloud.redhat.com/providers/deployment/impl.go +++ b/controllers/cloud.redhat.com/providers/deployment/impl.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" provutils "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/utils" "github.com/RedHatInsights/rhc-osdk-utils/utils" @@ -37,7 +38,23 @@ func (dp *deploymentProvider) makeDeployment(deployment crd.Deployment, app *crd return dp.Cache.Update(CoreDeployment, d) } -func setLocalAnnotations(env *crd.ClowdEnvironment, deployment *crd.Deployment, d *apps.Deployment, app *crd.ClowdApp) { +func (dp *deploymentProvider) makeStatefulSet(deployment crd.Deployment, app *crd.ClowdApp) error { + + s := &apps.StatefulSet{} + nn := app.GetDeploymentNamespacedName(&deployment) + + if err := dp.Cache.Create(CoreStatefulSet, nn, s); err != nil { + return err + } + + if err := initStatefulSet(app, dp.Env, s, nn, &deployment); err != nil { + return err + } + + return dp.Cache.Update(CoreStatefulSet, s) +} + +func setLocalAnnotations(env *crd.ClowdEnvironment, deployment *crd.Deployment, d client.Object, app *crd.ClowdApp) { if env.Spec.Providers.Web.Mode == "local" && (deployment.WebServices.Public.Enabled || bool(deployment.Web)) { annotations := map[string]string{ "clowder/authsidecar-image": provutils.GetCaddyImage(env), @@ -45,12 +62,11 @@ func setLocalAnnotations(env *crd.ClowdEnvironment, deployment *crd.Deployment, "clowder/authsidecar-port": strconv.Itoa(int(env.Spec.Providers.Web.Port)), "clowder/authsidecar-config": fmt.Sprintf("caddy-config-%s-%s", app.Name, deployment.Name), } - utils.UpdateAnnotations(&d.Spec.Template, annotations) + utils.UpdateAnnotations(d, annotations) } - } -func setMinReplicas(deployment *crd.Deployment, d *apps.Deployment) { +func setMinReplicasOnDeployment(deployment *crd.Deployment, d *apps.Deployment) { replicaCount := deployment.GetReplicaCount() // If deployment doesn't have minReplicas set, bail if replicaCount == nil { @@ -81,7 +97,39 @@ func setMinReplicas(deployment *crd.Deployment, d *apps.Deployment) { // Reset replicas to minReplicas if it somehow falls below minReplicas d.Spec.Replicas = replicaCount } +} + +func setMinReplicasOnStatefulSet(deployment *crd.Deployment, d *apps.StatefulSet) { + replicaCount := deployment.GetReplicaCount() + // If deployment doesn't have minReplicas set, bail + if replicaCount == nil { + return + } + + // Handle the special case of minReplicas being set to 0 used for manual scale down + if *replicaCount == 0 { + d.Spec.Replicas = utils.Int32Ptr(0) + return + } + + // No sense in running all these conditionals if desired state and observed state match + if d.Spec.Replicas != nil && (*d.Spec.Replicas >= *replicaCount) { + // if deployment has an autoscaler, just keep the replica count the same it currently is + // since it has at least the desired count + if deployment.HasAutoScaler() { + return + } + // reset the replica count when there is no autoscaler. this will scale down a deployment that + // has more than desired count + d.Spec.Replicas = replicaCount + } + // If the spec has nil replicas or the spec replicas are less than the deployment replicas + // then set the spec replicas to the deployment replicas + if d.Spec.Replicas == nil || (*d.Spec.Replicas < *replicaCount) { + // Reset replicas to minReplicas if it somehow falls below minReplicas + d.Spec.Replicas = replicaCount + } } func setDeploymentStrategy(deployment *crd.Deployment, d *apps.Deployment) { @@ -211,13 +259,11 @@ func initDeployment(app *crd.ClowdApp, env *crd.ClowdEnvironment, d *apps.Deploy d.Kind = "Deployment" - pod := deployment.PodSpec - utils.UpdateAnnotations(d, app.ObjectMeta.Annotations, deployment.Metadata.Annotations) setLocalAnnotations(env, deployment, d, app) - setMinReplicas(deployment, d) + setMinReplicasOnDeployment(deployment, d) d.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} d.Spec.Template.ObjectMeta.Labels = labels @@ -230,9 +276,54 @@ func initDeployment(app *crd.ClowdApp, env *crd.ClowdEnvironment, d *apps.Deploy } d.Spec.ProgressDeadlineSeconds = utils.Int32Ptr(600) - utils.UpdateAnnotations(&d.Spec.Template, pod.Metadata.Annotations) - setDeploymentStrategy(deployment, d) + podTemplateSpec, err := renderPodSpec(app, env, nn, deployment) + if err != nil { + return err + } + + for _, vol := range podTemplateSpec.Spec.Volumes { + v := vol + setRecreateDeploymentStrategyForPVCs(vol, d) + setVolumeSourceConfigMapDefaultMode(&v) + setVolumeSourceSecretDefaultMode(&v) + } + + d.Spec.Template = *podTemplateSpec + return nil +} + +func initStatefulSet(app *crd.ClowdApp, env *crd.ClowdEnvironment, s *apps.StatefulSet, nn types.NamespacedName, deployment *crd.Deployment) error { + labels := app.GetLabels() + labels["pod"] = nn.Name + app.SetObjectMeta(s, crd.Name(nn.Name), crd.Labels(labels)) + + s.Kind = "Deployment" + + utils.UpdateAnnotations(s, app.ObjectMeta.Annotations, deployment.Metadata.Annotations) + + setLocalAnnotations(env, deployment, s, app) + + setMinReplicasOnStatefulSet(deployment, s) + + s.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} + s.Spec.Template.ObjectMeta.Labels = labels + + podTemplateSpec, err := renderPodSpec(app, env, nn, deployment) + if err != nil { + return err + } + + s.Spec.Template = *podTemplateSpec + + return nil +} + +func renderPodSpec(app *crd.ClowdApp, env *crd.ClowdEnvironment, nn types.NamespacedName, deployment *crd.Deployment) (*core.PodTemplateSpec, error) { + // Everything from here is for the podspec and shoulr be ripped out + pod := deployment.PodSpec + podSpec := core.PodTemplateSpec{} + utils.UpdateAnnotations(&podSpec, pod.Metadata.Annotations) c := core.Container{ Name: nn.Name, @@ -257,31 +348,31 @@ func initDeployment(app *crd.ClowdApp, env *crd.ClowdEnvironment, d *apps.Deploy MountPath: "/cdapp/", }) - d.Spec.Template.Spec.Containers = []core.Container{c} + podSpec.Spec.Containers = []core.Container{c} ics, err := ProcessInitContainers(nn, &c, pod.InitContainers) if err != nil { - return err + return nil, err } if pod.MachinePool != "" { - d.Spec.Template.Spec.Tolerations = []core.Toleration{{ + podSpec.Spec.Tolerations = []core.Toleration{{ Key: pod.MachinePool, Effect: core.TaintEffectNoSchedule, Operator: core.TolerationOpEqual, Value: "true", }} } else { - d.Spec.Template.Spec.Tolerations = []core.Toleration{} + podSpec.Spec.Tolerations = []core.Toleration{} } - d.Spec.Template.Spec.InitContainers = ics + podSpec.Spec.InitContainers = ics - d.Spec.Template.Spec.TerminationGracePeriodSeconds = pod.TerminationGracePeriodSeconds + podSpec.Spec.TerminationGracePeriodSeconds = pod.TerminationGracePeriodSeconds - d.Spec.Template.Spec.Volumes = pod.Volumes - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, core.Volume{ + podSpec.Spec.Volumes = pod.Volumes + podSpec.Spec.Volumes = append(podSpec.Spec.Volumes, core.Volume{ Name: "config-secret", VolumeSource: core.VolumeSource{ Secret: &core.SecretVolumeSource{ @@ -291,16 +382,9 @@ func initDeployment(app *crd.ClowdApp, env *crd.ClowdEnvironment, d *apps.Deploy }, }) - for _, vol := range d.Spec.Template.Spec.Volumes { - v := vol - setRecreateDeploymentStrategyForPVCs(vol, d) - setVolumeSourceConfigMapDefaultMode(&v) - setVolumeSourceSecretDefaultMode(&v) - } - - ApplyPodAntiAffinity(&d.Spec.Template) + ApplyPodAntiAffinity(&podSpec) - return nil + return &podSpec, nil } func setRecreateDeploymentStrategyForPVCs(vol core.Volume, d *apps.Deployment) { From 7e081012d682d1d75867db71d2f2aaa8027b17fa Mon Sep 17 00:00:00 2001 From: Pete Savage Date: Fri, 10 Jan 2025 19:03:42 +0000 Subject: [PATCH 3/6] Final round of changes --- .../providers/deployment/provider.go | 8 ++++---- .../providers/metrics/impl.go | 11 +++++----- .../providers/servicemesh/default.go | 19 ++++++++++++++++++ .../providers/sidecar/default.go | 15 +++++++------- .../cloud.redhat.com/providers/web/default.go | 9 ++++----- .../cloud.redhat.com/providers/web/impl.go | 20 +++++++++---------- .../cloud.redhat.com/providers/web/local.go | 11 +++++----- 7 files changed, 53 insertions(+), 40 deletions(-) diff --git a/controllers/cloud.redhat.com/providers/deployment/provider.go b/controllers/cloud.redhat.com/providers/deployment/provider.go index 95c7d8b95..2537c7a45 100644 --- a/controllers/cloud.redhat.com/providers/deployment/provider.go +++ b/controllers/cloud.redhat.com/providers/deployment/provider.go @@ -7,7 +7,7 @@ import ( "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers" rc "github.com/RedHatInsights/rhc-osdk-utils/resourceCache" apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -24,7 +24,7 @@ func init() { providers.ProvidersRegistration.Register(GetDeployment, 0, ProvName) } -func GetPodTemplateFromObject(deployment *crd.Deployment, rc *rc.ObjectCache, nn types.NamespacedName) (*v1.PodTemplateSpec, error) { +func GetPodTemplateFromObject(deployment *crd.Deployment, rc *rc.ObjectCache, nn types.NamespacedName) (*core.PodTemplateSpec, error) { obj, err := GetClientObject(deployment, rc, nn) if err != nil { return nil, err @@ -55,10 +55,10 @@ func GetClientObject(deployment *crd.Deployment, rc *rc.ObjectCache, nn types.Na return d, nil } -func UpdatePodTemplate(deployment *crd.Deployment, podTemplate *v1.PodTemplateSpec, rc *rc.ObjectCache, nn types.NamespacedName) error { +func UpdatePodTemplate(deployment *crd.Deployment, podTemplate *core.PodTemplateSpec, rc *rc.ObjectCache, nn types.NamespacedName) error { if deployment.Stateful.Enabled { - ss := &apps.Deployment{} + ss := &apps.StatefulSet{} if err := rc.Get(CoreStatefulSet, ss, nn); err != nil { return err } diff --git a/controllers/cloud.redhat.com/providers/metrics/impl.go b/controllers/cloud.redhat.com/providers/metrics/impl.go index 882980aa7..f71f26b2d 100644 --- a/controllers/cloud.redhat.com/providers/metrics/impl.go +++ b/controllers/cloud.redhat.com/providers/metrics/impl.go @@ -9,7 +9,6 @@ import ( webProvider "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/web" prom "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -27,9 +26,9 @@ func makeMetrics(cache *rc.ObjectCache, deployment *crd.Deployment, app *crd.Clo return err } - d := &apps.Deployment{} - - if err := cache.Get(deployProvider.CoreDeployment, d, app.GetDeploymentNamespacedName(deployment)); err != nil { + nn := app.GetDeploymentNamespacedName(deployment) + pt, err := deployProvider.GetPodTemplateFromObject(deployment, cache, nn) + if err != nil { return err } @@ -44,7 +43,7 @@ func makeMetrics(cache *rc.ObjectCache, deployment *crd.Deployment, app *crd.Clo s.Spec.Ports = append(s.Spec.Ports, metricsPort) - d.Spec.Template.Spec.Containers[0].Ports = append(d.Spec.Template.Spec.Containers[0].Ports, + pt.Spec.Containers[0].Ports = append(pt.Spec.Containers[0].Ports, core.ContainerPort{ Name: "metrics", ContainerPort: port, @@ -56,7 +55,7 @@ func makeMetrics(cache *rc.ObjectCache, deployment *crd.Deployment, app *crd.Clo return err } - return cache.Update(deployProvider.CoreDeployment, d) + return deployProvider.UpdatePodTemplate(deployment, pt, cache, nn) } func createMetricsOnDeployments(cache *rc.ObjectCache, env *crd.ClowdEnvironment, app *crd.ClowdApp, c *config.AppConfig) error { diff --git a/controllers/cloud.redhat.com/providers/servicemesh/default.go b/controllers/cloud.redhat.com/providers/servicemesh/default.go index 45c04321a..d5721399b 100644 --- a/controllers/cloud.redhat.com/providers/servicemesh/default.go +++ b/controllers/cloud.redhat.com/providers/servicemesh/default.go @@ -48,5 +48,24 @@ func (ch *servicemeshProvider) Provide(_ *crd.ClowdApp) error { } } + ssList := apps.StatefulSetList{} + if err := ch.Cache.List(deployProvider.CoreStatefulSet, &ssList); err != nil { + return err + } + + for _, statefulset := range ssList.Items { + innerStatefulSet := statefulset + annotations := map[string]string{ + "sidecar.istio.io/inject": "true", + "traffic.sidecar.istio.io/excludeOutboundPorts": "443,9093,5432,10000", + } + utils.UpdateAnnotations(&innerStatefulSet.Spec.Template, annotations) + + err := ch.Cache.Update(deployProvider.CoreStatefulSet, &innerStatefulSet) + if err != nil { + return fmt.Errorf("could not update annotations: %w", err) + } + } + return nil } diff --git a/controllers/cloud.redhat.com/providers/sidecar/default.go b/controllers/cloud.redhat.com/providers/sidecar/default.go index 3dfe42e92..1ac6ada39 100644 --- a/controllers/cloud.redhat.com/providers/sidecar/default.go +++ b/controllers/cloud.redhat.com/providers/sidecar/default.go @@ -10,7 +10,6 @@ import ( provutils "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/utils" "github.com/RedHatInsights/rhc-osdk-utils/utils" - apps "k8s.io/api/apps/v1" batch "k8s.io/api/batch/v1" core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -31,9 +30,9 @@ func (sc *sidecarProvider) EnvProvide() error { func (sc *sidecarProvider) Provide(app *crd.ClowdApp) error { for _, deployment := range app.Spec.Deployments { innerDeployment := deployment - d := &apps.Deployment{} - - if err := sc.Cache.Get(deployProvider.CoreDeployment, d, app.GetDeploymentNamespacedName(&innerDeployment)); err != nil { + nn := app.GetDeploymentNamespacedName(&deployment) + pt, err := deployProvider.GetPodTemplateFromObject(&innerDeployment, sc.Cache, nn) + if err != nil { return err } @@ -43,15 +42,15 @@ func (sc *sidecarProvider) Provide(app *crd.ClowdApp) error { if sidecar.Enabled && sc.Env.Spec.Providers.Sidecars.TokenRefresher.Enabled { cont := getTokenRefresher(app.Name) if cont != nil { - d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, *cont) + pt.Spec.Containers = append(pt.Spec.Containers, *cont) } } case "otel-collector": if sidecar.Enabled && sc.Env.Spec.Providers.Sidecars.OtelCollector.Enabled { cont := getOtelCollector(app.Name) if cont != nil { - d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, *cont) - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, core.Volume{ + pt.Spec.Containers = append(pt.Spec.Containers, *cont) + pt.Spec.Volumes = append(pt.Spec.Volumes, core.Volume{ Name: fmt.Sprintf("%s-otel-config", app.Name), VolumeSource: core.VolumeSource{ ConfigMap: &core.ConfigMapVolumeSource{ @@ -69,7 +68,7 @@ func (sc *sidecarProvider) Provide(app *crd.ClowdApp) error { } } - if err := sc.Cache.Update(deployProvider.CoreDeployment, d); err != nil { + if err := deployProvider.UpdatePodTemplate(&deployment, pt, sc.Cache, nn); err != nil { return err } } diff --git a/controllers/cloud.redhat.com/providers/web/default.go b/controllers/cloud.redhat.com/providers/web/default.go index 11582bba9..83556550d 100644 --- a/controllers/cloud.redhat.com/providers/web/default.go +++ b/controllers/cloud.redhat.com/providers/web/default.go @@ -7,7 +7,6 @@ import ( provCronjob "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/cronjob" provDeploy "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/deployment" provutils "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/utils" - apps "k8s.io/api/apps/v1" batch "k8s.io/api/batch/v1" "github.com/RedHatInsights/rhc-osdk-utils/utils" @@ -50,16 +49,16 @@ func (web *webProvider) Provide(app *crd.ClowdApp) error { } if web.Env.Spec.Providers.Web.TLS.Enabled { - d := &apps.Deployment{} dnn := app.GetDeploymentNamespacedName(&innerDeployment) - if err := web.Cache.Get(provDeploy.CoreDeployment, d, dnn); err != nil { + pt, err := provDeploy.GetPodTemplateFromObject(&innerDeployment, web.Cache, dnn) + if err != nil { return errors.Wrap("getting core deployment", err) } - provutils.AddCertVolume(&d.Spec.Template.Spec, dnn.Name) + provutils.AddCertVolume(&pt.Spec, dnn.Name) - if err := web.Cache.Update(provDeploy.CoreDeployment, d); err != nil { + if err := provDeploy.UpdatePodTemplate(&innerDeployment, pt, web.Cache, dnn); err != nil { return errors.Wrap("updating core deployment", err) } } diff --git a/controllers/cloud.redhat.com/providers/web/impl.go b/controllers/cloud.redhat.com/providers/web/impl.go index 1a2f2fbe5..2571aff10 100644 --- a/controllers/cloud.redhat.com/providers/web/impl.go +++ b/controllers/cloud.redhat.com/providers/web/impl.go @@ -7,7 +7,6 @@ import ( deployProvider "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/deployment" provutils "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/utils" - apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -33,9 +32,8 @@ func makeService(cache *rc.ObjectCache, deployment *crd.Deployment, app *crd.Clo return err } - d := &apps.Deployment{} - - if err := cache.Get(deployProvider.CoreDeployment, d, app.GetDeploymentNamespacedName(deployment)); err != nil { + pt, err := deployProvider.GetPodTemplateFromObject(deployment, cache, app.GetDeploymentNamespacedName(deployment)) + if err != nil { return err } @@ -154,20 +152,20 @@ func makeService(cache *rc.ObjectCache, deployment *crd.Deployment, app *crd.Clo if err := generateCaddyConfigMap(cache, nn, app, pub, priv, pubPort, privPort, env); err != nil { return err } - populateSideCar(d, nn.Name, env.Spec.Providers.Web.TLS.Port, env.Spec.Providers.Web.TLS.PrivatePort, pub, priv) + populateSideCar(pt, nn.Name, env.Spec.Providers.Web.TLS.Port, env.Spec.Providers.Web.TLS.PrivatePort, pub, priv) setServiceTLSAnnotations(s, nn.Name) } } utils.MakeService(s, nn, map[string]string{"pod": nn.Name}, servicePorts, app, env.IsNodePort()) - d.Spec.Template.Spec.Containers[0].Ports = containerPorts + pt.Spec.Containers[0].Ports = containerPorts if err := cache.Update(CoreService, s); err != nil { return err } - return cache.Update(deployProvider.CoreDeployment, d) + return deployProvider.UpdatePodTemplate(deployment, pt, cache, nn) } func generateCaddyConfigMap(cache *rc.ObjectCache, nn types.NamespacedName, app *crd.ClowdApp, pub bool, priv bool, pubPort int32, privPort int32, env *crd.ClowdEnvironment) error { @@ -197,7 +195,7 @@ func generateCaddyConfigMap(cache *rc.ObjectCache, nn types.NamespacedName, app return cache.Update(CoreCaddyConfigMap, cm) } -func populateSideCar(d *apps.Deployment, name string, port int32, privatePort int32, pub bool, priv bool) { +func populateSideCar(pt *core.PodTemplateSpec, name string, port int32, privatePort int32, pub bool, priv bool) { ports := []core.ContainerPort{} if pub { ports = append(ports, core.ContainerPort{ @@ -248,13 +246,13 @@ func populateSideCar(d *apps.Deployment, name string, port int32, privatePort in VolumeSource: core.VolumeSource{ ConfigMap: &core.ConfigMapVolumeSource{ LocalObjectReference: core.LocalObjectReference{ - Name: caddyConfigName(d.Name), + Name: caddyConfigName(name), }, }, }, } - d.Spec.Template.Spec.Containers = append(d.Spec.Template.Spec.Containers, container) - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, caddyConfigVol, caddyTLSVol) + pt.Spec.Containers = append(pt.Spec.Containers, container) + pt.Spec.Volumes = append(pt.Spec.Volumes, caddyConfigVol, caddyTLSVol) } func setServiceTLSAnnotations(s *core.Service, name string) { diff --git a/controllers/cloud.redhat.com/providers/web/local.go b/controllers/cloud.redhat.com/providers/web/local.go index 48cc6cfc3..398e46a4b 100644 --- a/controllers/cloud.redhat.com/providers/web/local.go +++ b/controllers/cloud.redhat.com/providers/web/local.go @@ -13,7 +13,6 @@ import ( provDeploy "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/deployment" provutils "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/utils" - apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -147,23 +146,23 @@ func (web *localWebProvider) Provide(app *crd.ClowdApp) error { h.Write([]byte(jsonData)) hash := fmt.Sprintf("%x", h.Sum(nil)) - d := &apps.Deployment{} dnn := app.GetDeploymentNamespacedName(&innerDeployment) - if err := web.Cache.Get(provDeploy.CoreDeployment, d, dnn); err != nil { + pt, err := provDeploy.GetPodTemplateFromObject(&innerDeployment, web.Cache, dnn) + if err != nil { return err } if web.Env.Spec.Providers.Web.TLS.Enabled { - provutils.AddCertVolume(&d.Spec.Template.Spec, dnn.Name) + provutils.AddCertVolume(&pt.Spec, dnn.Name) } annotations := map[string]string{ "clowder/authsidecar-confighash": hash, } - utils.UpdateAnnotations(&d.Spec.Template, annotations) + utils.UpdateAnnotations(pt, annotations) - if err := web.Cache.Update(provDeploy.CoreDeployment, d); err != nil { + if err := provDeploy.UpdatePodTemplate(&innerDeployment, pt, web.Cache, dnn); err != nil { return err } From f4adfea75df437f306bf493eeb6d56bde6ad4c85 Mon Sep 17 00:00:00 2001 From: Pete Savage Date: Fri, 10 Jan 2025 21:20:15 +0000 Subject: [PATCH 4/6] Working unit tests --- Makefile | 2 +- .../providers/deployment/impl.go | 20 ++++++++----------- docs/api_reference.md | 17 ++++++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 0b29ff618..ce2869924 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ GO_CMD ?= go TEMPLATE_KUSTOMIZE ?= "deploy-kustomize.yaml" # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.28 +ENVTEST_K8S_VERSION = 1.31 # Image URL to use all building/pushing image targets ifeq ($(findstring -minikube,${MAKECMDGOALS}), -minikube) diff --git a/controllers/cloud.redhat.com/providers/deployment/impl.go b/controllers/cloud.redhat.com/providers/deployment/impl.go index 25b8067d6..4e1601d7a 100644 --- a/controllers/cloud.redhat.com/providers/deployment/impl.go +++ b/controllers/cloud.redhat.com/providers/deployment/impl.go @@ -277,19 +277,18 @@ func initDeployment(app *crd.ClowdApp, env *crd.ClowdEnvironment, d *apps.Deploy d.Spec.ProgressDeadlineSeconds = utils.Int32Ptr(600) setDeploymentStrategy(deployment, d) - podTemplateSpec, err := renderPodSpec(app, env, nn, deployment) + err := renderPodSpec(app, env, nn, deployment, &d.Spec.Template) if err != nil { return err } - for _, vol := range podTemplateSpec.Spec.Volumes { + for _, vol := range d.Spec.Template.Spec.Volumes { v := vol setRecreateDeploymentStrategyForPVCs(vol, d) setVolumeSourceConfigMapDefaultMode(&v) setVolumeSourceSecretDefaultMode(&v) } - d.Spec.Template = *podTemplateSpec return nil } @@ -309,21 +308,18 @@ func initStatefulSet(app *crd.ClowdApp, env *crd.ClowdEnvironment, s *apps.State s.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} s.Spec.Template.ObjectMeta.Labels = labels - podTemplateSpec, err := renderPodSpec(app, env, nn, deployment) + err := renderPodSpec(app, env, nn, deployment, &s.Spec.Template) if err != nil { return err } - s.Spec.Template = *podTemplateSpec - return nil } -func renderPodSpec(app *crd.ClowdApp, env *crd.ClowdEnvironment, nn types.NamespacedName, deployment *crd.Deployment) (*core.PodTemplateSpec, error) { +func renderPodSpec(app *crd.ClowdApp, env *crd.ClowdEnvironment, nn types.NamespacedName, deployment *crd.Deployment, podSpec *core.PodTemplateSpec) error { // Everything from here is for the podspec and shoulr be ripped out pod := deployment.PodSpec - podSpec := core.PodTemplateSpec{} - utils.UpdateAnnotations(&podSpec, pod.Metadata.Annotations) + utils.UpdateAnnotations(podSpec, pod.Metadata.Annotations) c := core.Container{ Name: nn.Name, @@ -353,7 +349,7 @@ func renderPodSpec(app *crd.ClowdApp, env *crd.ClowdEnvironment, nn types.Namesp ics, err := ProcessInitContainers(nn, &c, pod.InitContainers) if err != nil { - return nil, err + return err } if pod.MachinePool != "" { @@ -382,9 +378,9 @@ func renderPodSpec(app *crd.ClowdApp, env *crd.ClowdEnvironment, nn types.Namesp }, }) - ApplyPodAntiAffinity(&podSpec) + ApplyPodAntiAffinity(podSpec) - return &podSpec, nil + return nil } func setRecreateDeploymentStrategyForPVCs(vol core.Volume, d *apps.Deployment) { diff --git a/docs/api_reference.md b/docs/api_reference.md index f9fe2ce71..c6265845c 100644 --- a/docs/api_reference.md +++ b/docs/api_reference.md @@ -451,6 +451,7 @@ _Appears in:_ | `autoScalerSimple` _[AutoScalerSimple](#autoscalersimple)_ | | | | | `deploymentStrategy` _[DeploymentStrategy](#deploymentstrategy)_ | DeploymentStrategy allows the deployment strategy to be set only if the
deployment has no public service enabled | | | | `metadata` _[DeploymentMetadata](#deploymentmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `statefulSpec` _[StatefulSpec](#statefulspec)_ | | | | #### DeploymentConfig @@ -1421,6 +1422,22 @@ _Appears in:_ | `max` _integer_ | | | | +#### StatefulSpec + + + + + + + +_Appears in:_ +- [Deployment](#deployment) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `enabled` _boolean_ | | | | + + #### TLS From e89994ed81580e104c8d53117015a6aea4b534ee Mon Sep 17 00:00:00 2001 From: Pete Savage Date: Mon, 13 Jan 2025 14:21:45 +0000 Subject: [PATCH 5/6] Fixes for test failures --- .../cloud.redhat.com/providers/autoscaler/keda.go | 2 +- .../cloud.redhat.com/providers/deployment/impl.go | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/controllers/cloud.redhat.com/providers/autoscaler/keda.go b/controllers/cloud.redhat.com/providers/autoscaler/keda.go index bdc390d9e..867f9a4cc 100644 --- a/controllers/cloud.redhat.com/providers/autoscaler/keda.go +++ b/controllers/cloud.redhat.com/providers/autoscaler/keda.go @@ -42,7 +42,7 @@ func initAutoScaler(env *crd.ClowdEnvironment, app *crd.ClowdApp, obj client.Obj // Set up the watcher to watch the Deployment we created earlier. scalerSpec := keda.ScaledObjectSpec{ - ScaleTargetRef: &keda.ScaleTarget{Name: obj.GetName(), Kind: obj.GetObjectKind().GroupVersionKind().Kind, APIVersion: obj.GetObjectKind().GroupVersionKind().Version}, + ScaleTargetRef: &keda.ScaleTarget{Name: obj.GetName(), Kind: obj.GetObjectKind().GroupVersionKind().Kind, APIVersion: obj.GetObjectKind().GroupVersionKind().GroupVersion().String()}, PollingInterval: deployment.AutoScaler.PollingInterval, CooldownPeriod: deployment.AutoScaler.CooldownPeriod, Advanced: deployment.AutoScaler.Advanced, diff --git a/controllers/cloud.redhat.com/providers/deployment/impl.go b/controllers/cloud.redhat.com/providers/deployment/impl.go index 4e1601d7a..8d4384de5 100644 --- a/controllers/cloud.redhat.com/providers/deployment/impl.go +++ b/controllers/cloud.redhat.com/providers/deployment/impl.go @@ -12,7 +12,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" provutils "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/utils" "github.com/RedHatInsights/rhc-osdk-utils/utils" @@ -54,7 +53,7 @@ func (dp *deploymentProvider) makeStatefulSet(deployment crd.Deployment, app *cr return dp.Cache.Update(CoreStatefulSet, s) } -func setLocalAnnotations(env *crd.ClowdEnvironment, deployment *crd.Deployment, d client.Object, app *crd.ClowdApp) { +func setLocalAnnotations(env *crd.ClowdEnvironment, deployment *crd.Deployment, pt *core.PodTemplateSpec, app *crd.ClowdApp) { if env.Spec.Providers.Web.Mode == "local" && (deployment.WebServices.Public.Enabled || bool(deployment.Web)) { annotations := map[string]string{ "clowder/authsidecar-image": provutils.GetCaddyImage(env), @@ -62,7 +61,7 @@ func setLocalAnnotations(env *crd.ClowdEnvironment, deployment *crd.Deployment, "clowder/authsidecar-port": strconv.Itoa(int(env.Spec.Providers.Web.Port)), "clowder/authsidecar-config": fmt.Sprintf("caddy-config-%s-%s", app.Name, deployment.Name), } - utils.UpdateAnnotations(d, annotations) + utils.UpdateAnnotations(pt, annotations) } } @@ -261,8 +260,6 @@ func initDeployment(app *crd.ClowdApp, env *crd.ClowdEnvironment, d *apps.Deploy utils.UpdateAnnotations(d, app.ObjectMeta.Annotations, deployment.Metadata.Annotations) - setLocalAnnotations(env, deployment, d, app) - setMinReplicasOnDeployment(deployment, d) d.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} @@ -301,8 +298,6 @@ func initStatefulSet(app *crd.ClowdApp, env *crd.ClowdEnvironment, s *apps.State utils.UpdateAnnotations(s, app.ObjectMeta.Annotations, deployment.Metadata.Annotations) - setLocalAnnotations(env, deployment, s, app) - setMinReplicasOnStatefulSet(deployment, s) s.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} @@ -320,6 +315,7 @@ func renderPodSpec(app *crd.ClowdApp, env *crd.ClowdEnvironment, nn types.Namesp // Everything from here is for the podspec and shoulr be ripped out pod := deployment.PodSpec utils.UpdateAnnotations(podSpec, pod.Metadata.Annotations) + setLocalAnnotations(env, deployment, podSpec, app) c := core.Container{ Name: nn.Name, From 0d4cff0434737f40a53125ce31106606ddf39b86 Mon Sep 17 00:00:00 2001 From: Pete Savage Date: Tue, 14 Jan 2025 17:20:32 +0000 Subject: [PATCH 6/6] Add statefulset filterfuncs * Copy the filter funcs from @wcmitchell's PR :) --- .../cloud.redhat.com/clowdapp_controller.go | 1 + .../clowdenvironment_controller.go | 1 + controllers/cloud.redhat.com/filters.go | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/controllers/cloud.redhat.com/clowdapp_controller.go b/controllers/cloud.redhat.com/clowdapp_controller.go index e26a505bf..90c886927 100644 --- a/controllers/cloud.redhat.com/clowdapp_controller.go +++ b/controllers/cloud.redhat.com/clowdapp_controller.go @@ -203,6 +203,7 @@ func (r *ClowdAppReconciler) SetupWithManager(mgr ctrl.Manager) error { watchers := []Watcher{ {obj: &apps.Deployment{}, filter: deploymentFilter}, {obj: &core.Service{}, filter: generationOnlyFilter}, + {obj: &apps.StatefulSet{}, filter: statefulSetFilter}, {obj: &core.ConfigMap{}, filter: generationOnlyFilter}, {obj: &core.Secret{}, filter: alwaysFilter}, } diff --git a/controllers/cloud.redhat.com/clowdenvironment_controller.go b/controllers/cloud.redhat.com/clowdenvironment_controller.go index 38dd1744b..5ed24de7c 100644 --- a/controllers/cloud.redhat.com/clowdenvironment_controller.go +++ b/controllers/cloud.redhat.com/clowdenvironment_controller.go @@ -234,6 +234,7 @@ func (r *ClowdEnvironmentReconciler) SetupWithManager(mgr ctrl.Manager) error { {obj: &apps.Deployment{}, filter: deploymentFilter}, {obj: &core.Service{}, filter: alwaysFilter}, {obj: &core.Secret{}, filter: alwaysFilter}, + {obj: &apps.StatefulSet{}, filter: statefulSetFilter}, } if clowderconfig.LoadedConfig.Features.WatchStrimziResources { diff --git a/controllers/cloud.redhat.com/filters.go b/controllers/cloud.redhat.com/filters.go index 3293f8580..8b11088b6 100644 --- a/controllers/cloud.redhat.com/filters.go +++ b/controllers/cloud.redhat.com/filters.go @@ -66,6 +66,21 @@ func deploymentUpdateFunc(e event.UpdateEvent) bool { return false } +func statefulSetUpdateFunc(e event.UpdateEvent) bool { + objOld := e.ObjectOld.(*apps.StatefulSet) + objNew := e.ObjectNew.(*apps.StatefulSet) + if objNew.GetGeneration() != objOld.GetGeneration() { + return true + } + if (objOld.Status.AvailableReplicas != objNew.Status.AvailableReplicas) && (objNew.Status.AvailableReplicas == objNew.Status.ReadyReplicas) { + return true + } + if (objOld.Status.AvailableReplicas == objOld.Status.ReadyReplicas) && (objNew.Status.AvailableReplicas != objNew.Status.ReadyReplicas) { + return true + } + return false +} + func kafkaUpdateFunc(e event.UpdateEvent) bool { objOld := e.ObjectOld.(*strimzi.Kafka) objNew := e.ObjectNew.(*strimzi.Kafka) @@ -114,6 +129,10 @@ func deploymentFilter(logr logr.Logger, ctrlName string) HandlerFuncs { return genFilterFunc(deploymentUpdateFunc, logr, ctrlName) } +func statefulSetFilter(logr logr.Logger, ctrlName string) HandlerFuncs { + return genFilterFunc(statefulSetUpdateFunc, logr, ctrlName) +} + func kafkaFilter(logr logr.Logger, ctrlName string) HandlerFuncs { return genFilterFunc(kafkaUpdateFunc, logr, ctrlName) }