From 2695bc6459ad95ea536abcc7de8e376e93a9f94f Mon Sep 17 00:00:00 2001 From: Berkay Tekin Oz Date: Thu, 27 Feb 2025 15:00:22 +0300 Subject: [PATCH] feat(k8sd): introduce feature packages and update feature apply functions - Added the FeatureManifest struct that contains the required charts and images for the feature. - Features are updated to use feature manifests when reconciling through the feature controller. - Implementation modules under `features` are updated to contain sub modules per feature. - Values handling is separated from the `Apply` functions and is performed via helper functions instead. - Introduced helper `Values` types and related functions for merging values together using mergo. --- src/k8s/pkg/k8sd/features/calico/chart.go | 40 --- src/k8s/pkg/k8sd/features/calico/network.go | 143 -------- .../features/calico/{ => network}/internal.go | 8 +- .../calico/{ => network}/internal_test.go | 14 +- .../k8sd/features/calico/network/manifest.go | 47 +++ .../k8sd/features/calico/network/network.go | 97 ++++++ .../calico/{ => network}/network_test.go | 35 +- .../k8sd/features/calico/network/register.go | 28 ++ .../k8sd/features/calico/network/values.go | 163 +++++++++ src/k8s/pkg/k8sd/features/calico/register.go | 18 - src/k8s/pkg/k8sd/features/calico/status.go | 5 + src/k8s/pkg/k8sd/features/cilium/chart.go | 48 --- .../features/cilium/{ => gateway}/gateway.go | 82 +++-- .../cilium/{ => gateway}/gateway_test.go | 48 +-- .../k8sd/features/cilium/gateway/manifest.go | 33 ++ .../k8sd/features/cilium/gateway/values.go | 39 +++ .../features/cilium/{ => ingress}/ingress.go | 74 +++-- .../cilium/{ => ingress}/ingress_test.go | 48 +-- .../k8sd/features/cilium/ingress/manifest.go | 12 + .../k8sd/features/cilium/ingress/values.go | 60 ++++ .../cilium/{ => loadbalancer}/loadbalancer.go | 121 +++---- .../{ => loadbalancer}/loadbalancer_test.go | 36 +- .../features/cilium/loadbalancer/manifest.go | 23 ++ .../features/cilium/loadbalancer/values.go | 125 +++++++ src/k8s/pkg/k8sd/features/cilium/network.go | 310 ------------------ .../features/cilium/{ => network}/internal.go | 2 +- .../cilium/{ => network}/internal_test.go | 2 +- .../k8sd/features/cilium/network/manifest.go | 41 +++ .../k8sd/features/cilium/network/network.go | 150 +++++++++ .../cilium/{ => network}/network_test.go | 88 ++--- .../k8sd/features/cilium/network/register.go | 17 + .../k8sd/features/cilium/network/values.go | 216 ++++++++++++ src/k8s/pkg/k8sd/features/cilium/register.go | 8 - src/k8s/pkg/k8sd/features/cilium/util.go | 36 ++ src/k8s/pkg/k8sd/features/contour/chart.go | 61 ---- src/k8s/pkg/k8sd/features/contour/gateway.go | 136 -------- .../k8sd/features/contour/gateway/gateway.go | 89 +++++ .../contour/{ => gateway}/gateway_test.go | 37 ++- .../k8sd/features/contour/gateway/manifest.go | 41 +++ .../k8sd/features/contour/gateway/register.go | 17 + .../k8sd/features/contour/gateway/values.go | 35 ++ .../features/contour/{ => ingress}/ingress.go | 143 ++++---- .../contour/{ => ingress}/ingress_test.go | 51 +-- .../k8sd/features/contour/ingress/manifest.go | 55 ++++ .../k8sd/features/contour/ingress/register.go | 17 + .../k8sd/features/contour/ingress/values.go | 90 +++++ src/k8s/pkg/k8sd/features/contour/register.go | 10 - src/k8s/pkg/k8sd/features/contour/status.go | 6 + src/k8s/pkg/k8sd/features/contour/util.go | 72 ++++ src/k8s/pkg/k8sd/features/coredns/chart.go | 18 - .../features/coredns/{ => dns}/coredns.go | 79 ++--- .../coredns/{ => dns}/coredns_test.go | 33 +- .../pkg/k8sd/features/coredns/dns/manifest.go | 34 ++ .../pkg/k8sd/features/coredns/dns/register.go | 15 + .../pkg/k8sd/features/coredns/dns/values.go | 74 +++++ src/k8s/pkg/k8sd/features/coredns/register.go | 7 - .../k8sd/features/implementation_default.go | 20 +- .../k8sd/features/implementation_moonray.go | 21 +- src/k8s/pkg/k8sd/features/localpv/chart.go | 26 -- .../localpv/{ => local-storage}/localpv.go | 79 +++-- .../{ => local-storage}/localpv_test.go | 27 +- .../localpv/local-storage/manifest.go | 59 ++++ .../localpv/local-storage/register.go | 25 ++ .../features/localpv/local-storage/values.go | 85 +++++ src/k8s/pkg/k8sd/features/localpv/register.go | 13 - src/k8s/pkg/k8sd/features/localpv/status.go | 6 + src/k8s/pkg/k8sd/features/metallb/chart.go | 38 --- .../{ => loadbalancer}/loadbalancer.go | 100 ++---- .../{ => loadbalancer}/loadbalancer_test.go | 31 +- .../features/metallb/loadbalancer/manifest.go | 54 +++ .../features/metallb/loadbalancer/register.go | 19 ++ .../features/metallb/loadbalancer/values.go | 116 +++++++ src/k8s/pkg/k8sd/features/metallb/register.go | 9 - src/k8s/pkg/k8sd/features/metallb/status.go | 6 + .../pkg/k8sd/features/metrics-server/chart.go | 18 - .../k8sd/features/metrics-server/internal.go | 5 +- .../k8sd/features/metrics-server/manifest.go | 35 ++ .../features/metrics-server/metrics_server.go | 47 ++- .../k8sd/features/metrics-server/register.go | 4 +- .../k8sd/features/metrics-server/values.go | 65 ++++ src/k8s/pkg/k8sd/types/feature.go | 53 +++ 81 files changed, 2762 insertions(+), 1536 deletions(-) delete mode 100644 src/k8s/pkg/k8sd/features/calico/network.go rename src/k8s/pkg/k8sd/features/calico/{ => network}/internal.go (95%) rename src/k8s/pkg/k8sd/features/calico/{ => network}/internal_test.go (91%) create mode 100644 src/k8s/pkg/k8sd/features/calico/network/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/calico/network/network.go rename src/k8s/pkg/k8sd/features/calico/{ => network}/network_test.go (77%) create mode 100644 src/k8s/pkg/k8sd/features/calico/network/register.go create mode 100644 src/k8s/pkg/k8sd/features/calico/network/values.go rename src/k8s/pkg/k8sd/features/cilium/{ => gateway}/gateway.go (59%) rename src/k8s/pkg/k8sd/features/cilium/{ => gateway}/gateway_test.go (70%) create mode 100644 src/k8s/pkg/k8sd/features/cilium/gateway/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/cilium/gateway/values.go rename src/k8s/pkg/k8sd/features/cilium/{ => ingress}/ingress.go (60%) rename src/k8s/pkg/k8sd/features/cilium/{ => ingress}/ingress_test.go (65%) create mode 100644 src/k8s/pkg/k8sd/features/cilium/ingress/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/cilium/ingress/values.go rename src/k8s/pkg/k8sd/features/cilium/{ => loadbalancer}/loadbalancer.go (64%) rename src/k8s/pkg/k8sd/features/cilium/{ => loadbalancer}/loadbalancer_test.go (80%) create mode 100644 src/k8s/pkg/k8sd/features/cilium/loadbalancer/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/cilium/loadbalancer/values.go delete mode 100644 src/k8s/pkg/k8sd/features/cilium/network.go rename src/k8s/pkg/k8sd/features/cilium/{ => network}/internal.go (99%) rename src/k8s/pkg/k8sd/features/cilium/{ => network}/internal_test.go (99%) create mode 100644 src/k8s/pkg/k8sd/features/cilium/network/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/cilium/network/network.go rename src/k8s/pkg/k8sd/features/cilium/{ => network}/network_test.go (73%) create mode 100644 src/k8s/pkg/k8sd/features/cilium/network/register.go create mode 100644 src/k8s/pkg/k8sd/features/cilium/network/values.go create mode 100644 src/k8s/pkg/k8sd/features/cilium/util.go delete mode 100644 src/k8s/pkg/k8sd/features/contour/gateway.go create mode 100644 src/k8s/pkg/k8sd/features/contour/gateway/gateway.go rename src/k8s/pkg/k8sd/features/contour/{ => gateway}/gateway_test.go (69%) create mode 100644 src/k8s/pkg/k8sd/features/contour/gateway/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/contour/gateway/register.go create mode 100644 src/k8s/pkg/k8sd/features/contour/gateway/values.go rename src/k8s/pkg/k8sd/features/contour/{ => ingress}/ingress.go (54%) rename src/k8s/pkg/k8sd/features/contour/{ => ingress}/ingress_test.go (78%) create mode 100644 src/k8s/pkg/k8sd/features/contour/ingress/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/contour/ingress/register.go create mode 100644 src/k8s/pkg/k8sd/features/contour/ingress/values.go create mode 100644 src/k8s/pkg/k8sd/features/contour/status.go create mode 100644 src/k8s/pkg/k8sd/features/contour/util.go rename src/k8s/pkg/k8sd/features/coredns/{ => dns}/coredns.go (60%) rename src/k8s/pkg/k8sd/features/coredns/{ => dns}/coredns_test.go (77%) create mode 100644 src/k8s/pkg/k8sd/features/coredns/dns/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/coredns/dns/register.go create mode 100644 src/k8s/pkg/k8sd/features/coredns/dns/values.go rename src/k8s/pkg/k8sd/features/localpv/{ => local-storage}/localpv.go (52%) rename src/k8s/pkg/k8sd/features/localpv/{ => local-storage}/localpv_test.go (71%) create mode 100644 src/k8s/pkg/k8sd/features/localpv/local-storage/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/localpv/local-storage/register.go create mode 100644 src/k8s/pkg/k8sd/features/localpv/local-storage/values.go create mode 100644 src/k8s/pkg/k8sd/features/localpv/status.go rename src/k8s/pkg/k8sd/features/metallb/{ => loadbalancer}/loadbalancer.go (62%) rename src/k8s/pkg/k8sd/features/metallb/{ => loadbalancer}/loadbalancer_test.go (75%) create mode 100644 src/k8s/pkg/k8sd/features/metallb/loadbalancer/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/metallb/loadbalancer/register.go create mode 100644 src/k8s/pkg/k8sd/features/metallb/loadbalancer/values.go create mode 100644 src/k8s/pkg/k8sd/features/metallb/status.go create mode 100644 src/k8s/pkg/k8sd/features/metrics-server/manifest.go create mode 100644 src/k8s/pkg/k8sd/features/metrics-server/values.go create mode 100644 src/k8s/pkg/k8sd/types/feature.go diff --git a/src/k8s/pkg/k8sd/features/calico/chart.go b/src/k8s/pkg/k8sd/features/calico/chart.go index 256941e9f..74f2682a4 100644 --- a/src/k8s/pkg/k8sd/features/calico/chart.go +++ b/src/k8s/pkg/k8sd/features/calico/chart.go @@ -2,47 +2,7 @@ package calico import ( "embed" - - "github.com/canonical/k8s/pkg/client/helm" ) //go:embed all:charts var ChartFS embed.FS - -var ( - // ChartCalico represents manifests to deploy Calico. - ChartCalico = helm.InstallableChart{ - Name: "tigera-operator", - Version: "v3.28.0", - InstallName: "ck-network", - InstallNamespace: "tigera-operator", - } - - // Note: Tigera is the company behind Calico and the tigera-operator is the operator for Calico. - // TODO: use ROCKs instead of upstream - // imageRepo represents the repo to fetch the Calico CNI images. - imageRepo = "ghcr.io/canonical/k8s-snap" - - // calicoImageRepo represents the repo to fetch the calico images. - calicoImageRepo = "ghcr.io/canonical/k8s-snap/calico" - // CalicoTag represents the tag to use for the calico images. - CalicoTag = "v3.28.0" - - // tigeraOperatorImage represents the image to fetch for calico. - tigeraOperatorImage = "tigera/operator" - - // tigeraOperatorVersion is the version to use for the tigera-operator image. - tigeraOperatorVersion = "v1.34.0" - - // calicoCtlImage represents the image to fetch for calicoctl. - // TODO: use ROCKs instead of upstream - calicoCtlImage = "ghcr.io/canonical/k8s-snap/calico/ctl" - // calicoCtlTag represents the tag to use for the calicoctl image. - calicoCtlTag = "v3.28.0" - - // defaultEncapsulation represents the default defaultEncapsulation method to use for Calico. - defaultEncapsulation = "VXLAN" - - // defaultAPIServerEnabled determines if the Calico API server should be enabled. - defaultAPIServerEnabled = false -) diff --git a/src/k8s/pkg/k8sd/features/calico/network.go b/src/k8s/pkg/k8sd/features/calico/network.go deleted file mode 100644 index 1481ec742..000000000 --- a/src/k8s/pkg/k8sd/features/calico/network.go +++ /dev/null @@ -1,143 +0,0 @@ -package calico - -import ( - "context" - "fmt" - - "github.com/canonical/k8s/pkg/client/helm" - "github.com/canonical/k8s/pkg/k8sd/types" - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" - "github.com/canonical/microcluster/v2/state" -) - -const ( - EnabledMsg = "enabled" - DisabledMsg = "disabled" - deployFailedMsgTmpl = "Failed to deploy Calico, the error was: %v" - deleteFailedMsgTmpl = "Failed to delete Calico, the error was: %v" -) - -// ApplyNetwork will deploy Calico when network.Enabled is true. -// ApplyNetwork will remove Calico when network.Enabled is false. -// ApplyNetwork will always return a FeatureStatus indicating the current status of the -// deployment. -// ApplyNetwork returns an error if anything fails. The error is also wrapped in the .Message field of the -// returned FeatureStatus. -func ApplyNetwork(ctx context.Context, snap snap.Snap, m helm.Client, _ state.State, apiserver types.APIServer, network types.Network, annotations types.Annotations) (types.FeatureStatus, error) { - if !network.GetEnabled() { - if _, err := m.Apply(ctx, ChartCalico, helm.StateDeleted, nil); err != nil { - err = fmt.Errorf("failed to uninstall network: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CalicoTag, - Message: fmt.Sprintf(deleteFailedMsgTmpl, err), - }, err - } - - return types.FeatureStatus{ - Enabled: false, - Version: CalicoTag, - Message: DisabledMsg, - }, nil - } - - config, err := internalConfig(annotations) - if err != nil { - err = fmt.Errorf("failed to parse annotations: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CalicoTag, - Message: fmt.Sprintf(deployFailedMsgTmpl, err), - }, err - } - - podIpPools := []map[string]any{} - ipv4PodCIDR, ipv6PodCIDR, err := utils.SplitCIDRStrings(network.GetPodCIDR()) - if err != nil { - err = fmt.Errorf("invalid pod cidr: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CalicoTag, - Message: fmt.Sprintf(deployFailedMsgTmpl, err), - }, err - } - if ipv4PodCIDR != "" { - podIpPools = append(podIpPools, map[string]any{ - "name": "ipv4-ippool", - "cidr": ipv4PodCIDR, - "encapsulation": config.encapsulationV4, - }) - } - if ipv6PodCIDR != "" { - podIpPools = append(podIpPools, map[string]any{ - "name": "ipv6-ippool", - "cidr": ipv6PodCIDR, - "encapsulation": config.encapsulationV6, - }) - } - - serviceCIDRs := []string{} - ipv4ServiceCIDR, ipv6ServiceCIDR, err := utils.SplitCIDRStrings(network.GetServiceCIDR()) - if err != nil { - err = fmt.Errorf("invalid service cidr: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CalicoTag, - Message: fmt.Sprintf(deployFailedMsgTmpl, err), - }, err - } - if ipv4ServiceCIDR != "" { - serviceCIDRs = append(serviceCIDRs, ipv4ServiceCIDR) - } - if ipv6ServiceCIDR != "" { - serviceCIDRs = append(serviceCIDRs, ipv6ServiceCIDR) - } - - calicoNetworkValues := map[string]any{ - "ipPools": podIpPools, - } - - if config.autodetectionV4 != nil { - calicoNetworkValues["nodeAddressAutodetectionV4"] = config.autodetectionV4 - } - - if config.autodetectionV6 != nil { - calicoNetworkValues["nodeAddressAutodetectionV6"] = config.autodetectionV6 - } - - values := map[string]any{ - "tigeraOperator": map[string]any{ - "registry": imageRepo, - "image": tigeraOperatorImage, - "version": tigeraOperatorVersion, - }, - "calicoctl": map[string]any{ - "image": calicoCtlImage, - "tag": calicoCtlTag, - }, - "installation": map[string]any{ - "calicoNetwork": calicoNetworkValues, - "registry": imageRepo, - }, - "apiServer": map[string]any{ - "enabled": config.apiServerEnabled, - }, - "serviceCIDRs": serviceCIDRs, - } - - if _, err := m.Apply(ctx, ChartCalico, helm.StatePresent, values); err != nil { - err = fmt.Errorf("failed to enable network: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CalicoTag, - Message: fmt.Sprintf(deployFailedMsgTmpl, err), - }, err - } - - return types.FeatureStatus{ - Enabled: true, - Version: CalicoTag, - Message: EnabledMsg, - }, nil -} diff --git a/src/k8s/pkg/k8sd/features/calico/internal.go b/src/k8s/pkg/k8sd/features/calico/network/internal.go similarity index 95% rename from src/k8s/pkg/k8sd/features/calico/internal.go rename to src/k8s/pkg/k8sd/features/calico/network/internal.go index e3e48443d..340d0c402 100644 --- a/src/k8s/pkg/k8sd/features/calico/internal.go +++ b/src/k8s/pkg/k8sd/features/calico/network/internal.go @@ -1,4 +1,4 @@ -package calico +package network import ( "fmt" @@ -63,11 +63,7 @@ func parseAutodetectionAnnotations(annotations types.Annotations, autodetectionM } func internalConfig(annotations types.Annotations) (config, error) { - c := config{ - encapsulationV4: defaultEncapsulation, - encapsulationV6: defaultEncapsulation, - apiServerEnabled: defaultAPIServerEnabled, - } + c := config{} if v, ok := annotations.Get(apiv1_annotations.AnnotationAPIServerEnabled); ok { c.apiServerEnabled = v == "true" diff --git a/src/k8s/pkg/k8sd/features/calico/internal_test.go b/src/k8s/pkg/k8sd/features/calico/network/internal_test.go similarity index 91% rename from src/k8s/pkg/k8sd/features/calico/internal_test.go rename to src/k8s/pkg/k8sd/features/calico/network/internal_test.go index 208198315..cda5c9a7a 100644 --- a/src/k8s/pkg/k8sd/features/calico/internal_test.go +++ b/src/k8s/pkg/k8sd/features/calico/network/internal_test.go @@ -1,4 +1,4 @@ -package calico +package network import ( "testing" @@ -19,8 +19,8 @@ func TestInternalConfig(t *testing.T) { annotations: map[string]string{}, expectedConfig: config{ apiServerEnabled: false, - encapsulationV4: "VXLAN", - encapsulationV6: "VXLAN", + encapsulationV4: "", + encapsulationV6: "", }, expectError: false, }, @@ -33,7 +33,7 @@ func TestInternalConfig(t *testing.T) { expectedConfig: config{ apiServerEnabled: true, encapsulationV4: "IPIP", - encapsulationV6: "VXLAN", + encapsulationV6: "", }, expectError: false, }, @@ -53,7 +53,7 @@ func TestInternalConfig(t *testing.T) { expectedConfig: config{ apiServerEnabled: false, encapsulationV4: "VXLAN", - encapsulationV6: "VXLAN", + encapsulationV6: "", }, expectError: false, }, @@ -72,8 +72,8 @@ func TestInternalConfig(t *testing.T) { }, expectedConfig: config{ apiServerEnabled: false, - encapsulationV4: "VXLAN", - encapsulationV6: "VXLAN", + encapsulationV4: "", + encapsulationV6: "", autodetectionV4: map[string]any{ "cidrs": []string{"10.1.0.0/16", "2001:0db8::/32"}, }, diff --git a/src/k8s/pkg/k8sd/features/calico/network/manifest.go b/src/k8s/pkg/k8sd/features/calico/network/manifest.go new file mode 100644 index 000000000..0e71d2fce --- /dev/null +++ b/src/k8s/pkg/k8sd/features/calico/network/manifest.go @@ -0,0 +1,47 @@ +package network + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + CalicoChartName = "tigera-operator" + + CalicoImageName = "calico" + TigeraOperatorImageName = "tigera-operator" + CalicoCtlImageName = "calicoctl" +) + +var manifest = types.FeatureManifest{ + Name: "network", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + CalicoChartName: { + Name: "tigera-operator", + Version: "v3.28.0", + InstallName: "ck-network", + InstallNamespace: "tigera-operator", + }, + }, + + Images: map[string]types.Image{ + "calico": { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "calico", + Tag: "v3.28.0", + }, + "tigera-operator": { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "tigera/operator", + Tag: "v1.34.0", + }, + "calicoctl": { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "calico/ctl", + Tag: "v3.28.0", + }, + }, +} + +var FeatureNetwork types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/calico/network/network.go b/src/k8s/pkg/k8sd/features/calico/network/network.go new file mode 100644 index 000000000..16df7d09a --- /dev/null +++ b/src/k8s/pkg/k8sd/features/calico/network/network.go @@ -0,0 +1,97 @@ +package network + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/calico" + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/microcluster/v2/state" +) + +const ( + deployFailedMsgTmpl = "Failed to deploy Calico, the error was: %v" + deleteFailedMsgTmpl = "Failed to delete Calico, the error was: %v" +) + +// ApplyNetwork will deploy Calico when network.Enabled is true. +// ApplyNetwork will remove Calico when network.Enabled is false. +// ApplyNetwork will always return a FeatureStatus indicating the current status of the +// deployment. +// ApplyNetwork returns an error if anything fails. The error is also wrapped in the .Message field of the +// returned FeatureStatus. +func ApplyNetwork(ctx context.Context, snap snap.Snap, m helm.Client, _ state.State, apiserver types.APIServer, network types.Network, annotations types.Annotations) (types.FeatureStatus, error) { + calicoImage := FeatureNetwork.GetImage(CalicoImageName) + + if !network.GetEnabled() { + if _, err := m.Apply(ctx, FeatureNetwork.GetChart(CalicoChartName), helm.StateDeleted, nil); err != nil { + err = fmt.Errorf("failed to uninstall network: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: calicoImage.Tag, + Message: fmt.Sprintf(deleteFailedMsgTmpl, err), + }, err + } + + return types.FeatureStatus{ + Enabled: false, + Version: calicoImage.Tag, + Message: calico.DisabledMsg, + }, nil + } + + var values Values = map[string]any{} + + if err := values.ApplyDefaultValues(); err != nil { + err = fmt.Errorf("failed to apply default values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: calicoImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyImageOverrides(); err != nil { + err = fmt.Errorf("failed to calculate image overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: calicoImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyClusterConfiguration(ctx, nil, apiserver, network); err != nil { + err = fmt.Errorf("failed to calculate cluster config values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: calicoImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyAnnotations(annotations); err != nil { + err = fmt.Errorf("failed to calculate annotation overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: calicoImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if _, err := m.Apply(ctx, FeatureNetwork.GetChart(CalicoChartName), helm.StatePresent, values); err != nil { + err = fmt.Errorf("failed to enable network: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: calicoImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + return types.FeatureStatus{ + Enabled: true, + Version: calicoImage.Tag, + Message: calico.EnabledMsg, + }, nil +} diff --git a/src/k8s/pkg/k8sd/features/calico/network_test.go b/src/k8s/pkg/k8sd/features/calico/network/network_test.go similarity index 77% rename from src/k8s/pkg/k8sd/features/calico/network_test.go rename to src/k8s/pkg/k8sd/features/calico/network/network_test.go index 7b8586250..d4e92caf8 100644 --- a/src/k8s/pkg/k8sd/features/calico/network_test.go +++ b/src/k8s/pkg/k8sd/features/calico/network/network_test.go @@ -1,4 +1,4 @@ -package calico_test +package network_test import ( "context" @@ -10,6 +10,7 @@ import ( "github.com/canonical/k8s/pkg/client/helm/loader" helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/k8sd/features/calico" + calico_network "github.com/canonical/k8s/pkg/k8sd/features/calico/network" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" "github.com/canonical/k8s/pkg/utils" @@ -48,16 +49,16 @@ func TestDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&calico.ChartFS)) - status, err := calico.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, nil) + status, err := calico_network.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) - g.Expect(status.Version).To(Equal(calico.CalicoTag)) + g.Expect(status.Version).To(Equal(calico_network.FeatureNetwork.GetImage(calico_network.CalicoImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(calico.ChartCalico)) + g.Expect(callArgs.Chart).To(Equal(calico_network.FeatureNetwork.GetChart(calico_network.CalicoChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -78,16 +79,16 @@ func TestDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&calico.ChartFS)) - status, err := calico.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, nil) + status, err := calico_network.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(Equal(calico.DisabledMsg)) - g.Expect(status.Version).To(Equal(calico.CalicoTag)) + g.Expect(status.Version).To(Equal(calico_network.FeatureNetwork.GetImage(calico_network.CalicoImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(calico.ChartCalico)) + g.Expect(callArgs.Chart).To(Equal(calico_network.FeatureNetwork.GetChart(calico_network.CalicoChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -112,11 +113,11 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&calico.ChartFS)) - status, err := calico.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) + status, err := calico_network.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(calico.CalicoTag)) + g.Expect(status.Version).To(Equal(calico_network.FeatureNetwork.GetImage(calico_network.CalicoImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) }) t.Run("InvalidServiceCIDR", func(t *testing.T) { @@ -138,11 +139,11 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&calico.ChartFS)) - status, err := calico.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) + status, err := calico_network.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(calico.CalicoTag)) + g.Expect(status.Version).To(Equal(calico_network.FeatureNetwork.GetImage(calico_network.CalicoImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) }) t.Run("HelmApplyFails", func(t *testing.T) { @@ -167,16 +168,16 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&calico.ChartFS)) - status, err := calico.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) + status, err := calico_network.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) - g.Expect(status.Version).To(Equal(calico.CalicoTag)) + g.Expect(status.Version).To(Equal(calico_network.FeatureNetwork.GetImage(calico_network.CalicoImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(calico.ChartCalico)) + g.Expect(callArgs.Chart).To(Equal(calico_network.FeatureNetwork.GetChart(calico_network.CalicoChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateValues(t, callArgs.Values, network) }) @@ -199,16 +200,16 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&calico.ChartFS)) - status, err := calico.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) + status, err := calico_network.ApplyNetwork(context.Background(), snapM, mc, nil, apiserver, network, defaultAnnotations) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) g.Expect(status.Message).To(Equal(calico.EnabledMsg)) - g.Expect(status.Version).To(Equal(calico.CalicoTag)) + g.Expect(status.Version).To(Equal(calico_network.FeatureNetwork.GetImage(calico_network.CalicoImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(calico.ChartCalico)) + g.Expect(callArgs.Chart).To(Equal(calico_network.FeatureNetwork.GetChart(calico_network.CalicoChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateValues(t, callArgs.Values, network) }) diff --git a/src/k8s/pkg/k8sd/features/calico/network/register.go b/src/k8s/pkg/k8sd/features/calico/network/register.go new file mode 100644 index 000000000..9e348f828 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/calico/network/register.go @@ -0,0 +1,28 @@ +package network + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + tigeraOperatorImage := FeatureNetwork.GetImage(TigeraOperatorImageName) + calicoCtlImage := FeatureNetwork.GetImage(CalicoCtlImageName) + calicoImage := FeatureNetwork.GetImage(CalicoImageName) + + images.Register( + // Tigera images + fmt.Sprintf("%s/%s:%s", tigeraOperatorImage.Registry, tigeraOperatorImage.Repository, tigeraOperatorImage.Tag), + // Calico images + fmt.Sprintf("%s/apiserver:%s", calicoImage.GetURI(), calicoImage.Tag), + fmt.Sprintf("%s/cni:%s", calicoImage.GetURI(), calicoImage.Tag), + fmt.Sprintf("%s/csi:%s", calicoImage.GetURI(), calicoImage.Tag), + fmt.Sprintf("%s/ctl:%s", calicoCtlImage.GetURI(), calicoCtlImage.Tag), + fmt.Sprintf("%s/kube-controllers:%s", calicoImage.GetURI(), calicoImage.Tag), + fmt.Sprintf("%s/node:%s", calicoImage.GetURI(), calicoImage.Tag), + fmt.Sprintf("%s/node-driver-registrar:%s", calicoImage.GetURI(), calicoImage.Tag), + fmt.Sprintf("%s/pod2daemon-flexvol:%s", calicoImage.GetURI(), calicoImage.Tag), + fmt.Sprintf("%s/typha:%s", calicoImage.GetURI(), calicoImage.Tag), + ) +} diff --git a/src/k8s/pkg/k8sd/features/calico/network/values.go b/src/k8s/pkg/k8sd/features/calico/network/values.go new file mode 100644 index 000000000..1b3e8f457 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/calico/network/values.go @@ -0,0 +1,163 @@ +package network + +import ( + "context" + "fmt" + + "dario.cat/mergo" + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/utils" + "github.com/canonical/microcluster/v2/state" +) + +type Values map[string]any + +func (v Values) ApplyDefaultValues() error { + values := map[string]any{ + "apiServer": map[string]any{ + "enabled": false, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyImageOverrides() error { + tigeraOperatorImage := FeatureNetwork.GetImage(TigeraOperatorImageName) + calicoCtlImage := FeatureNetwork.GetImage(CalicoCtlImageName) + calicoImage := FeatureNetwork.GetImage(CalicoImageName) + + values := map[string]any{ + "tigeraOperator": map[string]any{ + "registry": tigeraOperatorImage.Registry, + "image": tigeraOperatorImage.Repository, + "version": tigeraOperatorImage.Tag, + }, + "calicoctl": map[string]any{ + "image": calicoCtlImage.GetURI(), + "tag": calicoCtlImage.Tag, + }, + "installation": map[string]any{ + "registry": calicoImage.Registry, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyClusterConfiguration(ctx context.Context, s state.State, apiserver types.APIServer, network types.Network) error { + podIpPools := []map[string]any{} + ipv4PodCIDR, ipv6PodCIDR, err := utils.SplitCIDRStrings(network.GetPodCIDR()) + if err != nil { + return fmt.Errorf("invalid pod cidr: %w", err) + } + if ipv4PodCIDR != "" { + podIpPools = append(podIpPools, map[string]any{ + "name": "ipv4-ippool", + "cidr": ipv4PodCIDR, + "encapsulation": "VXLAN", + }) + } + if ipv6PodCIDR != "" { + podIpPools = append(podIpPools, map[string]any{ + "name": "ipv6-ippool", + "cidr": ipv6PodCIDR, + "encapsulation": "VXLAN", + }) + } + + serviceCIDRs := []string{} + ipv4ServiceCIDR, ipv6ServiceCIDR, err := utils.SplitCIDRStrings(network.GetServiceCIDR()) + if err != nil { + return fmt.Errorf("invalid service cidr: %w", err) + } + if ipv4ServiceCIDR != "" { + serviceCIDRs = append(serviceCIDRs, ipv4ServiceCIDR) + } + if ipv6ServiceCIDR != "" { + serviceCIDRs = append(serviceCIDRs, ipv6ServiceCIDR) + } + + calicoNetworkValues := map[string]any{ + "ipPools": podIpPools, + } + + values := map[string]any{ + "installation": map[string]any{ + "calicoNetwork": calicoNetworkValues, + }, + "serviceCIDRs": serviceCIDRs, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyAnnotations(annotations types.Annotations) error { + config, err := internalConfig(annotations) + if err != nil { + return fmt.Errorf("failed to parse annotations: %w", err) + } + + // Merging slice of maps with mergo.Merge depends on the order of the elements in the slice. + // Instead of using mergo.Merge, we will manually merge the values. + installation, ok := v["installation"].(map[string]any) + if !ok { + return fmt.Errorf("installation values not found") + } + + calicoNetwork, ok := installation["calicoNetwork"].(map[string]any) + if !ok { + return fmt.Errorf("calicoNetwork values not found") + } + + ipPools, ok := calicoNetwork["ipPools"].([]map[string]any) + if !ok { + return fmt.Errorf("ipPools values not found") + } + + for _, pool := range ipPools { + if config.encapsulationV4 != "" && pool["name"] == "ipv4-ippool" { + pool["encapsulation"] = config.encapsulationV4 + } + if config.encapsulationV6 != "" && pool["name"] == "ipv6-ippool" { + pool["encapsulation"] = config.encapsulationV6 + } + } + + calicoNetworkValues := map[string]any{} + + if config.autodetectionV4 != nil { + calicoNetworkValues["nodeAddressAutodetectionV4"] = config.autodetectionV4 + } + + if config.autodetectionV6 != nil { + calicoNetworkValues["nodeAddressAutodetectionV6"] = config.autodetectionV6 + } + + values := map[string]any{ + "installation": map[string]any{ + "calicoNetwork": calicoNetworkValues, + }, + "apiServer": map[string]any{ + "enabled": config.apiServerEnabled, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/calico/register.go b/src/k8s/pkg/k8sd/features/calico/register.go index 1abdb9594..054b2bddc 100644 --- a/src/k8s/pkg/k8sd/features/calico/register.go +++ b/src/k8s/pkg/k8sd/features/calico/register.go @@ -1,27 +1,9 @@ package calico import ( - "fmt" - "github.com/canonical/k8s/pkg/k8sd/charts" - "github.com/canonical/k8s/pkg/k8sd/images" ) func init() { - images.Register( - // Tigera images - fmt.Sprintf("%s/%s:%s", imageRepo, tigeraOperatorImage, tigeraOperatorVersion), - // Calico images - fmt.Sprintf("%s/apiserver:%s", calicoImageRepo, CalicoTag), - fmt.Sprintf("%s/cni:%s", calicoImageRepo, CalicoTag), - fmt.Sprintf("%s/csi:%s", calicoImageRepo, CalicoTag), - fmt.Sprintf("%s/ctl:%s", calicoImageRepo, calicoCtlTag), - fmt.Sprintf("%s/kube-controllers:%s", calicoImageRepo, CalicoTag), - fmt.Sprintf("%s/node:%s", calicoImageRepo, CalicoTag), - fmt.Sprintf("%s/node-driver-registrar:%s", calicoImageRepo, CalicoTag), - fmt.Sprintf("%s/pod2daemon-flexvol:%s", calicoImageRepo, CalicoTag), - fmt.Sprintf("%s/typha:%s", calicoImageRepo, CalicoTag), - ) - charts.Register(&ChartFS) } diff --git a/src/k8s/pkg/k8sd/features/calico/status.go b/src/k8s/pkg/k8sd/features/calico/status.go index 8cafae4d4..1e79e49ae 100644 --- a/src/k8s/pkg/k8sd/features/calico/status.go +++ b/src/k8s/pkg/k8sd/features/calico/status.go @@ -8,6 +8,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + EnabledMsg = "enabled" + DisabledMsg = "disabled" +) + // CheckNetwork checks the status of the Calico pods in the Kubernetes cluster. // We verify that the tigera-operator and calico-node pods are Ready and in Running state. func CheckNetwork(ctx context.Context, snap snap.Snap) error { diff --git a/src/k8s/pkg/k8sd/features/cilium/chart.go b/src/k8s/pkg/k8sd/features/cilium/chart.go index be0a89a78..92c60d80c 100644 --- a/src/k8s/pkg/k8sd/features/cilium/chart.go +++ b/src/k8s/pkg/k8sd/features/cilium/chart.go @@ -2,55 +2,7 @@ package cilium import ( "embed" - - "github.com/canonical/k8s/pkg/client/helm" ) //go:embed all:charts var ChartFS embed.FS - -var ( - // ChartCilium represents manifests to deploy Cilium. - ChartCilium = helm.InstallableChart{ - Name: "cilium", - Version: "1.16.7", - InstallName: "ck-network", - InstallNamespace: "kube-system", - } - - // ChartCiliumLoadBalancer represents manifests to deploy Cilium LoadBalancer resources. - ChartCiliumLoadBalancer = helm.InstallableChart{ - Name: "ck-loadbalancer", - Version: "0.1.1", - InstallName: "ck-loadbalancer", - InstallNamespace: "kube-system", - } - - // chartGateway represents manifests to deploy Gateway API CRDs. - chartGateway = helm.InstallableChart{ - Name: "gateway-api", - Version: "1.1.1", - InstallName: "ck-gateway", - InstallNamespace: "kube-system", - } - - // chartGatewayClass represents a manifest to deploy a GatewayClass called ck-gateway. - chartGatewayClass = helm.InstallableChart{ - Name: "ck-gateway-cilium", - Version: "0.1.0", - InstallName: "ck-gateway-class", - InstallNamespace: "default", - } - - // ciliumAgentImageRepo represents the image to use for cilium-agent. - ciliumAgentImageRepo = "ghcr.io/canonical/cilium" - - // CiliumAgentImageTag is the tag to use for the cilium-agent image. - CiliumAgentImageTag = "1.16.7-ck0" - - // ciliumOperatorImageRepo is the image to use for cilium-operator. - ciliumOperatorImageRepo = "ghcr.io/canonical/cilium-operator" - - // ciliumOperatorImageTag is the tag to use for the cilium-operator image. - ciliumOperatorImageTag = "1.16.7-ck0" -) diff --git a/src/k8s/pkg/k8sd/features/cilium/gateway.go b/src/k8s/pkg/k8sd/features/cilium/gateway/gateway.go similarity index 59% rename from src/k8s/pkg/k8sd/features/cilium/gateway.go rename to src/k8s/pkg/k8sd/features/cilium/gateway/gateway.go index 5bace019b..39791f8ee 100644 --- a/src/k8s/pkg/k8sd/features/cilium/gateway.go +++ b/src/k8s/pkg/k8sd/features/cilium/gateway/gateway.go @@ -1,10 +1,12 @@ -package cilium +package gateway import ( "context" "fmt" "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" ) @@ -24,38 +26,51 @@ const ( // returned FeatureStatus. func ApplyGateway(ctx context.Context, snap snap.Snap, m helm.Client, gateway types.Gateway, network types.Network, _ types.Annotations) (types.FeatureStatus, error) { if gateway.GetEnabled() { - return enableGateway(ctx, snap, m) + return enableGateway(ctx, snap, m, gateway) } return disableGateway(ctx, snap, m, network) } -func enableGateway(ctx context.Context, snap snap.Snap, m helm.Client) (types.FeatureStatus, error) { +func enableGateway(ctx context.Context, snap snap.Snap, m helm.Client, gateway types.Gateway) (types.FeatureStatus, error) { + ciliumAgentImageTag := cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag + // Install Gateway API CRDs - if _, err := m.Apply(ctx, chartGateway, helm.StatePresent, nil); err != nil { + if _, err := m.Apply(ctx, FeatureGateway.GetChart(GatewayChartName), helm.StatePresent, nil); err != nil { err = fmt.Errorf("failed to install Gateway API CRDs: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), }, err } // Apply our GatewayClass named ck-gateway - if _, err := m.Apply(ctx, chartGatewayClass, helm.StatePresent, nil); err != nil { + if _, err := m.Apply(ctx, FeatureGateway.GetChart(GatewayClassChartName), helm.StatePresent, nil); err != nil { err = fmt.Errorf("failed to install Gateway API GatewayClass: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, + Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), + }, err + } + + var ciliumValues CiliumValues = map[string]any{} + + if err := ciliumValues.ApplyClusterConfiguration(gateway); err != nil { + err = fmt.Errorf("failed to apply cluster configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), }, err } - changed, err := m.Apply(ctx, ChartCilium, helm.StateUpgradeOnly, map[string]any{"gatewayAPI": map[string]any{"enabled": true}}) + changed, err := m.Apply(ctx, cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName), helm.StateUpgradeOnly, ciliumValues) if err != nil { err = fmt.Errorf("failed to upgrade Gateway API cilium configuration: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), }, err } @@ -63,55 +78,68 @@ func enableGateway(ctx context.Context, snap snap.Snap, m helm.Client) (types.Fe if !changed { return types.FeatureStatus{ Enabled: true, - Version: CiliumAgentImageTag, - Message: EnabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.EnabledMsg, }, nil } - if err := rolloutRestartCilium(ctx, snap, 3); err != nil { + if err := cilium.RolloutRestartCilium(ctx, snap, 3); err != nil { err = fmt.Errorf("failed to rollout restart cilium to enable Gateway API: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), }, err } return types.FeatureStatus{ Enabled: true, - Version: CiliumAgentImageTag, - Message: EnabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.EnabledMsg, }, nil } func disableGateway(ctx context.Context, snap snap.Snap, m helm.Client, network types.Network) (types.FeatureStatus, error) { + ciliumAgentImageTag := cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag + // Delete our GatewayClass named ck-gateway - if _, err := m.Apply(ctx, chartGatewayClass, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureGateway.GetChart(GatewayClassChartName), helm.StateDeleted, nil); err != nil { err = fmt.Errorf("failed to delete Gateway API GatewayClass: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, + Message: fmt.Sprintf(GatewayDeleteFailedMsgTmpl, err), + }, err + } + + var ciliumValues CiliumValues = map[string]any{} + + if err := ciliumValues.ApplyDisableConfiguration(); err != nil { + err = fmt.Errorf("failed to apply disable configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeleteFailedMsgTmpl, err), }, err } - changed, err := m.Apply(ctx, ChartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), map[string]any{"gatewayAPI": map[string]any{"enabled": false}}) + changed, err := m.Apply(ctx, cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName), helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), ciliumValues) if err != nil { err = fmt.Errorf("failed to delete Gateway API cilium configuration: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeleteFailedMsgTmpl, err), }, err } // Remove Gateway CRDs if the Gateway feature is disabled. // This is done after the Cilium update as cilium requires the CRDs to be present for cleanups. - if _, err := m.Apply(ctx, chartGateway, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureGateway.GetChart(GatewayChartName), helm.StateDeleted, nil); err != nil { err = fmt.Errorf("failed to delete Gateway API CRDs: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeleteFailedMsgTmpl, err), }, err } @@ -119,23 +147,23 @@ func disableGateway(ctx context.Context, snap snap.Snap, m helm.Client, network if !changed { return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, - Message: DisabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.DisabledMsg, }, nil } - if err := rolloutRestartCilium(ctx, snap, 3); err != nil { + if err := cilium.RolloutRestartCilium(ctx, snap, 3); err != nil { err = fmt.Errorf("failed to rollout restart cilium to disable Gateway API: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), }, err } return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, - Message: DisabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.DisabledMsg, }, nil } diff --git a/src/k8s/pkg/k8sd/features/cilium/gateway_test.go b/src/k8s/pkg/k8sd/features/cilium/gateway/gateway_test.go similarity index 70% rename from src/k8s/pkg/k8sd/features/cilium/gateway_test.go rename to src/k8s/pkg/k8sd/features/cilium/gateway/gateway_test.go index eb6bccb8e..7abfd075e 100644 --- a/src/k8s/pkg/k8sd/features/cilium/gateway_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/gateway/gateway_test.go @@ -1,4 +1,4 @@ -package cilium_test +package gateway_test import ( "context" @@ -11,6 +11,8 @@ import ( helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/client/kubernetes" "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_gateway "github.com/canonical/k8s/pkg/k8sd/features/cilium/gateway" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -40,13 +42,13 @@ func TestGatewayEnabled(t *testing.T) { mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.GatewayDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_gateway.GatewayDeployFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -67,15 +69,15 @@ func TestGatewayEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(status.Message).To(Equal(cilium.EnabledMsg)) helmCiliumArgs := helmM.ApplyCalledWith[2] - g.Expect(helmCiliumArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(helmCiliumArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(helmCiliumArgs.State).To(Equal(helm.StateUpgradeOnly)) g.Expect(helmCiliumArgs.Values["gatewayAPI"].(map[string]any)["enabled"]).To(BeTrue()) }) @@ -101,12 +103,12 @@ func TestGatewayEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.GatewayDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_gateway.GatewayDeployFailedMsgTmpl, err))) }) t.Run("Success", func(t *testing.T) { @@ -143,11 +145,11 @@ func TestGatewayEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(status.Message).To(Equal(cilium.EnabledMsg)) }) } @@ -171,13 +173,13 @@ func TestGatewayDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.GatewayDeleteFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_gateway.GatewayDeleteFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -197,14 +199,14 @@ func TestGatewayDisabled(t *testing.T) { Enabled: ptr.To(false), } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(status.Message).To(Equal(cilium.DisabledMsg)) helmCiliumArgs := helmM.ApplyCalledWith[1] - g.Expect(helmCiliumArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(helmCiliumArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(helmCiliumArgs.State).To(Equal(helm.StateDeleted)) g.Expect(helmCiliumArgs.Values["gatewayAPI"].(map[string]any)["enabled"]).To(BeFalse()) }) @@ -229,11 +231,11 @@ func TestGatewayDisabled(t *testing.T) { Enabled: ptr.To(false), } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.GatewayDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_gateway.GatewayDeployFailedMsgTmpl, err))) }) t.Run("Success", func(t *testing.T) { @@ -269,11 +271,11 @@ func TestGatewayDisabled(t *testing.T) { Enabled: ptr.To(false), } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := cilium_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(status.Message).To(Equal(cilium.DisabledMsg)) }) } diff --git a/src/k8s/pkg/k8sd/features/cilium/gateway/manifest.go b/src/k8s/pkg/k8sd/features/cilium/gateway/manifest.go new file mode 100644 index 000000000..1bbb84320 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/gateway/manifest.go @@ -0,0 +1,33 @@ +package gateway + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + GatewayChartName = "gateway-api" + GatewayClassChartName = "ck-gateway-cilium" +) + +var manifest = types.FeatureManifest{ + Name: "gateway", + Version: "1.0.0", + + Charts: map[string]helm.InstallableChart{ + GatewayChartName: { + Name: "gateway-api", + Version: "1.1.1", + InstallName: "ck-gateway", + InstallNamespace: "kube-system", + }, + GatewayClassChartName: { + Name: "ck-gateway-cilium", + Version: "0.1.0", + InstallName: "ck-gateway-class", + InstallNamespace: "default", + }, + }, +} + +var FeatureGateway types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/cilium/gateway/values.go b/src/k8s/pkg/k8sd/features/cilium/gateway/values.go new file mode 100644 index 000000000..52f34c7ad --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/gateway/values.go @@ -0,0 +1,39 @@ +package gateway + +import ( + "fmt" + + "dario.cat/mergo" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +type CiliumValues cilium_network.Values + +func (v CiliumValues) ApplyClusterConfiguration(gateway types.Gateway) error { + values := map[string]any{ + "gatewayAPI": map[string]any{ + "enabled": gateway.GetEnabled(), + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge cluster configuration: %w", err) + } + + return nil +} + +func (v CiliumValues) ApplyDisableConfiguration() error { + values := map[string]any{ + "gatewayAPI": map[string]any{ + "enabled": false, + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge disable configuration: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/cilium/ingress.go b/src/k8s/pkg/k8sd/features/cilium/ingress/ingress.go similarity index 60% rename from src/k8s/pkg/k8sd/features/cilium/ingress.go rename to src/k8s/pkg/k8sd/features/cilium/ingress/ingress.go index b3a3bf80d..7619fab29 100644 --- a/src/k8s/pkg/k8sd/features/cilium/ingress.go +++ b/src/k8s/pkg/k8sd/features/cilium/ingress/ingress.go @@ -1,10 +1,12 @@ -package cilium +package ingress import ( "context" "fmt" "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" ) @@ -31,43 +33,53 @@ const ( // ApplyIngress returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. func ApplyIngress(ctx context.Context, snap snap.Snap, m helm.Client, ingress types.Ingress, network types.Network, _ types.Annotations) (types.FeatureStatus, error) { - var values map[string]any + ciliumAgentImageTag := cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag + + var ciliumValues CiliumValues = map[string]any{} + if ingress.GetEnabled() { - values = map[string]any{ - "ingressController": map[string]any{ - IngressOptionEnabled: true, - IngressOptionLoadBalancerMode: IngressOptionLoadBalancerModeShared, - IngressOptionDefaultSecretNamespace: IngressOptionDefaultSecretNamespaceKubeSystem, - IngressOptionDefaultSecretName: ingress.GetDefaultTLSSecret(), - IngressOptionEnableProxyProtocol: ingress.GetEnableProxyProtocol(), - }, + if err := ciliumValues.ApplyDefaultValues(); err != nil { + err = fmt.Errorf("failed to apply default values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImageTag, + Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), + }, err + } + + if err := ciliumValues.ApplyClusterConfiguration(ingress); err != nil { + err = fmt.Errorf("failed to apply cluster configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImageTag, + Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), + }, err } } else { - values = map[string]any{ - "ingressController": map[string]any{ - IngressOptionEnabled: false, - IngressOptionLoadBalancerMode: "", - IngressOptionDefaultSecretNamespace: "", - IngressOptionDefaultSecretName: "", - IngressOptionEnableProxyProtocol: false, - }, + if err := ciliumValues.ApplyDisableConfiguration(); err != nil { + err = fmt.Errorf("failed to apply disable configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImageTag, + Message: fmt.Sprintf(IngressDeleteFailedMsgTmpl, err), + }, err } } - changed, err := m.Apply(ctx, ChartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), values) + changed, err := m.Apply(ctx, cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName), helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), ciliumValues) if err != nil { if network.GetEnabled() { err = fmt.Errorf("failed to enable ingress: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err } else { err = fmt.Errorf("failed to disable ingress: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(IngressDeleteFailedMsgTmpl, err), }, err } @@ -77,14 +89,14 @@ func ApplyIngress(ctx context.Context, snap snap.Snap, m helm.Client, ingress ty if ingress.GetEnabled() { return types.FeatureStatus{ Enabled: true, - Version: CiliumAgentImageTag, - Message: EnabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.EnabledMsg, }, nil } else { return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, - Message: DisabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.DisabledMsg, }, nil } } @@ -92,23 +104,23 @@ func ApplyIngress(ctx context.Context, snap snap.Snap, m helm.Client, ingress ty if !ingress.GetEnabled() { return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, - Message: DisabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.DisabledMsg, }, nil } - if err := rolloutRestartCilium(ctx, snap, 3); err != nil { + if err := cilium.RolloutRestartCilium(ctx, snap, 3); err != nil { err = fmt.Errorf("failed to rollout restart cilium to apply ingress: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err } return types.FeatureStatus{ Enabled: true, - Version: CiliumAgentImageTag, - Message: EnabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.EnabledMsg, }, nil } diff --git a/src/k8s/pkg/k8sd/features/cilium/ingress_test.go b/src/k8s/pkg/k8sd/features/cilium/ingress/ingress_test.go similarity index 65% rename from src/k8s/pkg/k8sd/features/cilium/ingress_test.go rename to src/k8s/pkg/k8sd/features/cilium/ingress/ingress_test.go index 95f182cc4..47e79b95e 100644 --- a/src/k8s/pkg/k8sd/features/cilium/ingress_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/ingress/ingress_test.go @@ -1,4 +1,4 @@ -package cilium_test +package ingress_test import ( "context" @@ -10,6 +10,8 @@ import ( helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/client/kubernetes" "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_ingress "github.com/canonical/k8s/pkg/k8sd/features/cilium/ingress" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -38,13 +40,13 @@ func TestIngress(t *testing.T) { name: "HelmFailNetworkEnabled", networkEnabled: true, helmErr: applyErr, - statusMsg: fmt.Sprintf(cilium.IngressDeployFailedMsgTmpl, fmt.Errorf("failed to enable ingress: %w", applyErr)), + statusMsg: fmt.Sprintf(cilium_ingress.IngressDeployFailedMsgTmpl, fmt.Errorf("failed to enable ingress: %w", applyErr)), statusEnabled: false, }, { name: "HelmFailNetworkDisabled", networkEnabled: false, - statusMsg: fmt.Sprintf(cilium.IngressDeleteFailedMsgTmpl, fmt.Errorf("failed to disable ingress: %w", applyErr)), + statusMsg: fmt.Sprintf(cilium_ingress.IngressDeleteFailedMsgTmpl, fmt.Errorf("failed to disable ingress: %w", applyErr)), statusEnabled: false, helmErr: applyErr, }, @@ -97,7 +99,7 @@ func TestIngress(t *testing.T) { EnableProxyProtocol: ptr.To(tc.enableProxyProtocol), } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) + status, err := cilium_ingress.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) if tc.helmErr == nil { g.Expect(err).To(Not(HaveOccurred())) @@ -106,11 +108,11 @@ func TestIngress(t *testing.T) { } g.Expect(status.Enabled).To(Equal(tc.statusEnabled)) g.Expect(status.Message).To(Equal(tc.statusMsg)) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) validateIngressValues(g, callArgs.Values, ingress) }) } @@ -144,16 +146,16 @@ func TestIngressRollout(t *testing.T) { Enabled: ptr.To(true), } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) + status, err := cilium_ingress.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.IngressDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_ingress.IngressDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) validateIngressValues(g, callArgs.Values, ingress) }) @@ -190,16 +192,16 @@ func TestIngressRollout(t *testing.T) { Enabled: ptr.To(true), } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) + status, err := cilium_ingress.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) g.Expect(err).To(Not(HaveOccurred())) g.Expect(status.Enabled).To(BeTrue()) g.Expect(status.Message).To(Equal(cilium.EnabledMsg)) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) validateIngressValues(g, callArgs.Values, ingress) }) } @@ -208,16 +210,16 @@ func validateIngressValues(g Gomega, values map[string]any, ingress types.Ingres ingressController, ok := values["ingressController"].(map[string]any) g.Expect(ok).To(BeTrue()) if ingress.GetEnabled() { - g.Expect(ingressController[cilium.IngressOptionEnabled]).To(BeTrue()) - g.Expect(ingressController[cilium.IngressOptionLoadBalancerMode]).To(Equal(cilium.IngressOptionLoadBalancerModeShared)) - g.Expect(ingressController[cilium.IngressOptionDefaultSecretNamespace]).To(Equal(cilium.IngressOptionDefaultSecretNamespaceKubeSystem)) - g.Expect(ingressController[cilium.IngressOptionDefaultSecretName]).To(Equal(ingress.GetDefaultTLSSecret())) - g.Expect(ingressController[cilium.IngressOptionEnableProxyProtocol]).To(Equal(ingress.GetEnableProxyProtocol())) + g.Expect(ingressController[cilium_ingress.IngressOptionEnabled]).To(BeTrue()) + g.Expect(ingressController[cilium_ingress.IngressOptionLoadBalancerMode]).To(Equal(cilium_ingress.IngressOptionLoadBalancerModeShared)) + g.Expect(ingressController[cilium_ingress.IngressOptionDefaultSecretNamespace]).To(Equal(cilium_ingress.IngressOptionDefaultSecretNamespaceKubeSystem)) + g.Expect(ingressController[cilium_ingress.IngressOptionDefaultSecretName]).To(Equal(ingress.GetDefaultTLSSecret())) + g.Expect(ingressController[cilium_ingress.IngressOptionEnableProxyProtocol]).To(Equal(ingress.GetEnableProxyProtocol())) } else { - g.Expect(ingressController[cilium.IngressOptionEnabled]).To(BeFalse()) - g.Expect(ingressController[cilium.IngressOptionLoadBalancerMode]).To(Equal("")) - g.Expect(ingressController[cilium.IngressOptionDefaultSecretNamespace]).To(Equal("")) - g.Expect(ingressController[cilium.IngressOptionDefaultSecretName]).To(Equal("")) - g.Expect(ingressController[cilium.IngressOptionEnableProxyProtocol]).To(BeFalse()) + g.Expect(ingressController[cilium_ingress.IngressOptionEnabled]).To(BeFalse()) + g.Expect(ingressController[cilium_ingress.IngressOptionLoadBalancerMode]).To(Equal("")) + g.Expect(ingressController[cilium_ingress.IngressOptionDefaultSecretNamespace]).To(Equal("")) + g.Expect(ingressController[cilium_ingress.IngressOptionDefaultSecretName]).To(Equal("")) + g.Expect(ingressController[cilium_ingress.IngressOptionEnableProxyProtocol]).To(BeFalse()) } } diff --git a/src/k8s/pkg/k8sd/features/cilium/ingress/manifest.go b/src/k8s/pkg/k8sd/features/cilium/ingress/manifest.go new file mode 100644 index 000000000..991ea97c4 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/ingress/manifest.go @@ -0,0 +1,12 @@ +package ingress + +import ( + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var manifest = types.FeatureManifest{ + Name: "ingress", + Version: "1.0.0", +} + +var FeatureIngress types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/cilium/ingress/values.go b/src/k8s/pkg/k8sd/features/cilium/ingress/values.go new file mode 100644 index 000000000..c73df827c --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/ingress/values.go @@ -0,0 +1,60 @@ +package ingress + +import ( + "fmt" + + "dario.cat/mergo" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +type CiliumValues cilium_network.Values + +func (v CiliumValues) ApplyDefaultValues() error { + values := map[string]any{ + "ingressController": map[string]any{ + "loadbalancerMode": "shared", + "defaultSecretNamespace": "kube-system", + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v CiliumValues) ApplyClusterConfiguration(ingress types.Ingress) error { + values := map[string]any{ + "ingressController": map[string]any{ + IngressOptionEnabled: ingress.GetEnabled(), + IngressOptionDefaultSecretName: ingress.GetDefaultTLSSecret(), + IngressOptionEnableProxyProtocol: ingress.GetEnableProxyProtocol(), + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge cluster configuration: %w", err) + } + + return nil +} + +func (v CiliumValues) ApplyDisableConfiguration() error { + values := map[string]any{ + "ingressController": map[string]any{ + IngressOptionEnabled: false, + IngressOptionLoadBalancerMode: "", + IngressOptionDefaultSecretNamespace: "", + IngressOptionDefaultSecretName: "", + IngressOptionEnableProxyProtocol: false, + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge disable configuration: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/loadbalancer.go similarity index 64% rename from src/k8s/pkg/k8sd/features/cilium/loadbalancer.go rename to src/k8s/pkg/k8sd/features/cilium/loadbalancer/loadbalancer.go index ea224d891..d6f47b30e 100644 --- a/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go +++ b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/loadbalancer.go @@ -1,10 +1,12 @@ -package cilium +package loadbalancer import ( "context" "fmt" "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" "github.com/canonical/k8s/pkg/utils/control" @@ -25,19 +27,21 @@ const ( // ApplyLoadBalancer returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadbalancer types.LoadBalancer, network types.Network, _ types.Annotations) (types.FeatureStatus, error) { + ciliumAgentImageTag := cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag + if !loadbalancer.GetEnabled() { if err := disableLoadBalancer(ctx, snap, m, network); err != nil { err = fmt.Errorf("failed to disable LoadBalancer: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(LbDeleteFailedMsgTmpl, err), }, err } return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, - Message: DisabledMsg, + Version: ciliumAgentImageTag, + Message: cilium.DisabledMsg, }, nil } @@ -45,7 +49,7 @@ func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadb err = fmt.Errorf("failed to enable LoadBalancer: %w", err) return types.FeatureStatus{ Enabled: false, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(LbDeployFailedMsgTmpl, err), }, err } @@ -54,73 +58,53 @@ func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadb case loadbalancer.GetBGPMode(): return types.FeatureStatus{ Enabled: true, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(lbEnabledMsgTmpl, "BGP"), }, nil case loadbalancer.GetL2Mode(): return types.FeatureStatus{ Enabled: true, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(lbEnabledMsgTmpl, "L2"), }, nil default: return types.FeatureStatus{ Enabled: true, - Version: CiliumAgentImageTag, + Version: ciliumAgentImageTag, Message: fmt.Sprintf(lbEnabledMsgTmpl, "Unknown"), }, nil } } func disableLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, network types.Network) error { - if _, err := m.Apply(ctx, ChartCiliumLoadBalancer, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureLoadBalancer.GetChart(LoadbalancerChartName), helm.StateDeleted, nil); err != nil { return fmt.Errorf("failed to uninstall LoadBalancer manifests: %w", err) } - values := map[string]any{ - "l2announcements": map[string]any{ - "enabled": false, - }, - "bgpControlPlane": map[string]any{ - "enabled": false, - }, - "externalIPs": map[string]any{ - "enabled": false, - }, - // https://docs.cilium.io/en/v1.14/network/l2-announcements/#sizing-client-rate-limit - // Setting back to defaults - "k8sClientRateLimit": map[string]any{ - "qps": 5, - "burst": 10, - }, - } - - if _, err := m.Apply(ctx, ChartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), values); err != nil { + var ciliumValues CiliumValues = map[string]any{} + + if err := ciliumValues.ApplyDisableConfiguration(); err != nil { + return fmt.Errorf("failed to apply disable configuration: %w", err) + } + + if _, err := m.Apply(ctx, cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName), helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), ciliumValues); err != nil { return fmt.Errorf("failed to refresh network to apply LoadBalancer configuration: %w", err) } return nil } func enableLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadbalancer types.LoadBalancer, network types.Network) error { - networkValues := map[string]any{ - "l2announcements": map[string]any{ - "enabled": loadbalancer.GetL2Mode(), - }, - "bgpControlPlane": map[string]any{ - "enabled": loadbalancer.GetBGPMode(), - }, - "externalIPs": map[string]any{ - "enabled": true, - }, - // https://docs.cilium.io/en/v1.14/network/l2-announcements/#sizing-client-rate-limit - // Assuming for 50 LB services - "k8sClientRateLimit": map[string]any{ - "qps": 10, - "burst": 20, - }, - } - - changed, err := m.Apply(ctx, ChartCilium, helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), networkValues) + var ciliumValues CiliumValues = map[string]any{} + + if err := ciliumValues.ApplyDefaultValues(); err != nil { + return fmt.Errorf("failed to apply default values: %w", err) + } + + if err := ciliumValues.ApplyClusterConfiguration(loadbalancer); err != nil { + return fmt.Errorf("failed to apply cluster configuration: %w", err) + } + + changed, err := m.Apply(ctx, cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName), helm.StateUpgradeOnlyOrDeleted(network.GetEnabled()), ciliumValues) if err != nil { return fmt.Errorf("failed to update Cilium configuration for LoadBalancer: %w", err) } @@ -129,36 +113,17 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, load return fmt.Errorf("failed to wait for required Cilium CRDs to be available: %w", err) } - cidrs := []map[string]any{} - for _, cidr := range loadbalancer.GetCIDRs() { - cidrs = append(cidrs, map[string]any{"cidr": cidr}) - } - for _, ipRange := range loadbalancer.GetIPRanges() { - cidrs = append(cidrs, map[string]any{"start": ipRange.Start, "stop": ipRange.Stop}) - } - - values := map[string]any{ - "driver": "cilium", - "l2": map[string]any{ - "enabled": loadbalancer.GetL2Mode(), - "interfaces": loadbalancer.GetL2Interfaces(), - }, - "ipPool": map[string]any{ - "cidrs": cidrs, - }, - "bgp": map[string]any{ - "enabled": loadbalancer.GetBGPMode(), - "localASN": loadbalancer.GetBGPLocalASN(), - "neighbors": []map[string]any{ - { - "peerAddress": loadbalancer.GetBGPPeerAddress(), - "peerASN": loadbalancer.GetBGPPeerASN(), - "peerPort": loadbalancer.GetBGPPeerPort(), - }, - }, - }, - } - if _, err := m.Apply(ctx, ChartCiliumLoadBalancer, helm.StatePresent, values); err != nil { + var values Values = map[string]any{} + + if err := values.ApplyDefaultValues(); err != nil { + return fmt.Errorf("failed to apply default values: %w", err) + } + + if err := values.ApplyClusterConfiguration(loadbalancer); err != nil { + return fmt.Errorf("failed to apply cluster configuration: %w", err) + } + + if _, err := m.Apply(ctx, FeatureLoadBalancer.GetChart(LoadbalancerChartName), helm.StatePresent, values); err != nil { return fmt.Errorf("failed to apply LoadBalancer configuration: %w", err) } @@ -166,7 +131,7 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, load return nil } - if err := rolloutRestartCilium(ctx, snap, 3); err != nil { + if err := cilium.RolloutRestartCilium(ctx, snap, 3); err != nil { return fmt.Errorf("failed to rollout restart cilium to apply LoadBalancer configuration: %w", err) } return nil diff --git a/src/k8s/pkg/k8sd/features/cilium/loadbalancer_test.go b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/loadbalancer_test.go similarity index 80% rename from src/k8s/pkg/k8sd/features/cilium/loadbalancer_test.go rename to src/k8s/pkg/k8sd/features/cilium/loadbalancer/loadbalancer_test.go index a67c3b94e..d5adbf3cf 100644 --- a/src/k8s/pkg/k8sd/features/cilium/loadbalancer_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/loadbalancer_test.go @@ -1,4 +1,4 @@ -package cilium_test +package loadbalancer_test import ( "context" @@ -11,6 +11,8 @@ import ( helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/client/kubernetes" "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_loadbalancer "github.com/canonical/k8s/pkg/k8sd/features/cilium/loadbalancer" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -41,16 +43,16 @@ func TestLoadBalancerDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) + status, err := cilium_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.LbDeleteFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_loadbalancer.LbDeleteFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCiliumLoadBalancer)) + g.Expect(callArgs.Chart).To(Equal(cilium_loadbalancer.FeatureLoadBalancer.GetChart(cilium_loadbalancer.LoadbalancerChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -72,22 +74,22 @@ func TestLoadBalancerDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, networkCfg, nil) + status, err := cilium_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, networkCfg, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(Equal(cilium.DisabledMsg)) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(2)) firstCallArgs := helmM.ApplyCalledWith[0] - g.Expect(firstCallArgs.Chart).To(Equal(cilium.ChartCiliumLoadBalancer)) + g.Expect(firstCallArgs.Chart).To(Equal(cilium_loadbalancer.FeatureLoadBalancer.GetChart(cilium_loadbalancer.LoadbalancerChartName))) g.Expect(firstCallArgs.State).To(Equal(helm.StateDeleted)) g.Expect(firstCallArgs.Values).To(BeNil()) // checking helm apply for network since it's enabled secondCallArgs := helmM.ApplyCalledWith[1] - g.Expect(secondCallArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(secondCallArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(secondCallArgs.State).To(Equal(helm.StateUpgradeOnlyOrDeleted(networkCfg.GetEnabled()))) }) } @@ -116,16 +118,16 @@ func TestLoadBalancerEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, networkCfg, nil) + status, err := cilium_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, networkCfg, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.LbDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_loadbalancer.LbDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(callArgs.State).To(Equal(helm.StateUpgradeOnlyOrDeleted(networkCfg.GetEnabled()))) l2announcements, ok := callArgs.Values["l2announcements"].(map[string]any) g.Expect(ok).To(BeTrue()) @@ -218,17 +220,17 @@ func TestLoadBalancerEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, networkCfg, nil) + status, err := cilium_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, networkCfg, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(status.Message).To(Equal(tc.statusMessage)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(2)) firstCallArgs := helmM.ApplyCalledWith[0] - g.Expect(firstCallArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(firstCallArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(firstCallArgs.State).To(Equal(helm.StateUpgradeOnlyOrDeleted(networkCfg.GetEnabled()))) l2announcements, ok := firstCallArgs.Values["l2announcements"].(map[string]any) g.Expect(ok).To(BeTrue()) @@ -238,7 +240,7 @@ func TestLoadBalancerEnabled(t *testing.T) { g.Expect(bgpControlPlane["enabled"]).To(Equal(lbCfg.GetBGPMode())) secondCallArgs := helmM.ApplyCalledWith[1] - g.Expect(secondCallArgs.Chart).To(Equal(cilium.ChartCiliumLoadBalancer)) + g.Expect(secondCallArgs.Chart).To(Equal(cilium_loadbalancer.FeatureLoadBalancer.GetChart(cilium_loadbalancer.LoadbalancerChartName))) g.Expect(secondCallArgs.State).To(Equal(helm.StatePresent)) validateLoadBalancerValues(t, secondCallArgs.Values, lbCfg) diff --git a/src/k8s/pkg/k8sd/features/cilium/loadbalancer/manifest.go b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/manifest.go new file mode 100644 index 000000000..d4c3eaaf2 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/manifest.go @@ -0,0 +1,23 @@ +package loadbalancer + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var LoadbalancerChartName = "ck-loadbalancer" + +var manifest = types.FeatureManifest{ + Name: "loadbalancer", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + LoadbalancerChartName: { + Name: "ck-loadbalancer", + Version: "0.1.1", + InstallName: "ck-loadbalancer", + InstallNamespace: "kube-system", + }, + }, +} + +var FeatureLoadBalancer types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/cilium/loadbalancer/values.go b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/values.go new file mode 100644 index 000000000..7b1c0a50e --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/loadbalancer/values.go @@ -0,0 +1,125 @@ +package loadbalancer + +import ( + "fmt" + + "dario.cat/mergo" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +type CiliumValues cilium_network.Values + +func (v CiliumValues) ApplyDefaultValues() error { + values := map[string]any{ + "externalIPs": map[string]any{ + "enabled": true, + }, + // https://docs.cilium.io/en/v1.14/network/l2-announcements/#sizing-client-rate-limit + // Assuming for 50 LB services + "k8sClientRateLimit": map[string]any{ + "qps": 10, + "burst": 20, + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v CiliumValues) ApplyClusterConfiguration(loadbalancer types.LoadBalancer) error { + values := map[string]any{ + "l2announcements": map[string]any{ + "enabled": loadbalancer.GetL2Mode(), + }, + "bgpControlPlane": map[string]any{ + "enabled": loadbalancer.GetBGPMode(), + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +func (v CiliumValues) ApplyDisableConfiguration() error { + values := map[string]any{ + "l2announcements": map[string]any{ + "enabled": false, + }, + "bgpControlPlane": map[string]any{ + "enabled": false, + }, + "externalIPs": map[string]any{ + "enabled": false, + }, + // https://docs.cilium.io/en/v1.14/network/l2-announcements/#sizing-client-rate-limit + // Setting back to defaults + "k8sClientRateLimit": map[string]any{ + "qps": 5, + "burst": 10, + }, + } + + if err := mergo.Merge(&v, CiliumValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +type Values map[string]any + +func (v Values) ApplyDefaultValues() error { + values := map[string]any{ + "driver": "cilium", + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyClusterConfiguration(loadbalancer types.LoadBalancer) error { + cidrs := []map[string]any{} + for _, cidr := range loadbalancer.GetCIDRs() { + cidrs = append(cidrs, map[string]any{"cidr": cidr}) + } + for _, ipRange := range loadbalancer.GetIPRanges() { + cidrs = append(cidrs, map[string]any{"start": ipRange.Start, "stop": ipRange.Stop}) + } + + values := map[string]any{ + "l2": map[string]any{ + "enabled": loadbalancer.GetL2Mode(), + "interfaces": loadbalancer.GetL2Interfaces(), + }, + "ipPool": map[string]any{ + "cidrs": cidrs, + }, + "bgp": map[string]any{ + "enabled": loadbalancer.GetBGPMode(), + "localASN": loadbalancer.GetBGPLocalASN(), + "neighbors": []map[string]any{ + { + "peerAddress": loadbalancer.GetBGPPeerAddress(), + "peerASN": loadbalancer.GetBGPPeerASN(), + "peerPort": loadbalancer.GetBGPPeerPort(), + }, + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/cilium/network.go b/src/k8s/pkg/k8sd/features/cilium/network.go deleted file mode 100644 index dba660af0..000000000 --- a/src/k8s/pkg/k8sd/features/cilium/network.go +++ /dev/null @@ -1,310 +0,0 @@ -package cilium - -import ( - "context" - "fmt" - "net" - "strings" - - "github.com/canonical/k8s/pkg/client/helm" - "github.com/canonical/k8s/pkg/k8sd/types" - "github.com/canonical/k8s/pkg/log" - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils" - "github.com/canonical/k8s/pkg/utils/control" - "github.com/canonical/microcluster/v2/state" -) - -const ( - NetworkDeleteFailedMsgTmpl = "Failed to delete Cilium Network, the error was: %v" - NetworkDeployFailedMsgTmpl = "Failed to deploy Cilium Network, the error was: %v" -) - -// required for unittests. -var ( - GetMountPath = utils.GetMountPath - GetMountPropagationType = utils.GetMountPropagationType -) - -// ApplyNetwork will deploy Cilium when network.Enabled is true. -// ApplyNetwork will remove Cilium when network.Enabled is false. -// ApplyNetwork requires that bpf and cgroups2 are already mounted and available when running under strict snap confinement. If they are not, it will fail (since Cilium will not have the required permissions to mount them). -// ApplyNetwork requires that `/sys` is mounted as a shared mount when running under classic snap confinement. This is to ensure that Cilium will be able to automatically mount bpf and cgroups2 on the pods. -// ApplyNetwork will always return a FeatureStatus indicating the current status of the -// deployment. -// ApplyNetwork returns an error if anything fails. The error is also wrapped in the .Message field of the -// returned FeatureStatus. -func ApplyNetwork(ctx context.Context, snap snap.Snap, m helm.Client, s state.State, apiserver types.APIServer, network types.Network, annotations types.Annotations) (types.FeatureStatus, error) { - if !network.GetEnabled() { - if _, err := m.Apply(ctx, ChartCilium, helm.StateDeleted, nil); err != nil { - err = fmt.Errorf("failed to uninstall network: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeleteFailedMsgTmpl, err), - }, err - } - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: DisabledMsg, - }, nil - } - - config, err := internalConfig(annotations) - if err != nil { - err = fmt.Errorf("failed to parse annotations: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - c, err := s.Leader() - if err != nil { - err = fmt.Errorf("failed to get leader client: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - clusterMembers, err := c.GetClusterMembers(ctx) - if err != nil { - err = fmt.Errorf("failed to get cluster members: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - localhostAddress, err := utils.DetermineLocalhostAddress(clusterMembers) - if err != nil { - err = fmt.Errorf("failed to determine localhost address: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - nodeIP := net.ParseIP(s.Address().Hostname()) - if nodeIP == nil { - err = fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname()) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - defaultCidr, err := utils.FindCIDRForIP(nodeIP) - if err != nil { - err = fmt.Errorf("failed to find cidr of default interface: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - ipv4CIDR, ipv6CIDR, err := utils.SplitCIDRStrings(network.GetPodCIDR()) - if err != nil { - err = fmt.Errorf("invalid kube-proxy --cluster-cidr value: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - ciliumNodePortValues := map[string]any{ - "enabled": true, - // kube-proxy also binds to the same port for health checks so we need to disable it - "enableHealthCheck": false, - } - - if config.directRoutingDevice != "" { - ciliumNodePortValues["directRoutingDevice"] = config.directRoutingDevice - } - - bpfValues := map[string]any{} - if config.vlanBPFBypass != nil { - bpfValues["vlanBypass"] = config.vlanBPFBypass - } - - values := map[string]any{ - "bpf": bpfValues, - "image": map[string]any{ - "repository": ciliumAgentImageRepo, - "tag": CiliumAgentImageTag, - "useDigest": false, - }, - "socketLB": map[string]any{ - "enabled": true, - }, - "cni": map[string]any{ - "confPath": "/etc/cni/net.d", - "binPath": "/opt/cni/bin", - "exclusive": config.cniExclusive, - }, - "sctp": map[string]any{ - "enabled": config.sctpEnabled, - }, - "operator": map[string]any{ - "replicas": 1, - "image": map[string]any{ - "repository": ciliumOperatorImageRepo, - "tag": ciliumOperatorImageTag, - "useDigest": false, - }, - }, - "ipv4": map[string]any{ - "enabled": ipv4CIDR != "", - }, - "ipv6": map[string]any{ - "enabled": ipv6CIDR != "", - }, - "ipam": map[string]any{ - "operator": map[string]any{ - "clusterPoolIPv4PodCIDRList": ipv4CIDR, - "clusterPoolIPv6PodCIDRList": ipv6CIDR, - }, - }, - "envoy": map[string]any{ - "enabled": false, // 1.16+ installs envoy as a standalone daemonset by default if not explicitly disabled - }, - // https://docs.cilium.io/en/v1.15/network/kubernetes/kubeproxy-free/#kube-proxy-hybrid-modes - "nodePort": ciliumNodePortValues, - "disableEnvoyVersionCheck": true, - // socketLB requires an endpoint to the apiserver that's not managed by the kube-proxy - // so we point to the localhost:secureport to talk to either the kube-apiserver or the kube-apiserver-proxy - "k8sServiceHost": strings.Trim(localhostAddress, "[]"), // Cilium already adds the brackets for ipv6 addresses, so we need to remove them - "k8sServicePort": apiserver.GetSecurePort(), - // This flag enables the runtime device detection which is set to true by default in Cilium 1.16+ - "enableRuntimeDeviceDetection": true, - } - - // If we are deploying with IPv6 only, we need to set the routing mode to native - if ipv4CIDR == "" && ipv6CIDR != "" { - values["routingMode"] = "native" - values["ipv6NativeRoutingCIDR"] = defaultCidr - values["autoDirectNodeRoutes"] = true - } - - if config.devices != "" { - values["devices"] = config.devices - } - - if snap.Strict() { - bpfMnt, err := GetMountPath("bpf") - if err != nil { - err = fmt.Errorf("failed to get bpf mount path: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - cgrMnt, err := GetMountPath("cgroup2") - if err != nil { - err = fmt.Errorf("failed to get cgroup2 mount path: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - values["bpf"] = map[string]any{ - "autoMount": map[string]any{ - "enabled": false, - }, - "root": bpfMnt, - } - values["cgroup"] = map[string]any{ - "autoMount": map[string]any{ - "enabled": false, - }, - "hostRoot": cgrMnt, - } - } else { - pt, err := GetMountPropagationType("/sys") - if err != nil { - err = fmt.Errorf("failed to get mount propagation type for /sys: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - if pt == utils.MountPropagationPrivate { - onLXD, err := snap.OnLXD(ctx) - if err != nil { - logger := log.FromContext(ctx) - logger.Error(err, "Failed to check if running on LXD") - } - if onLXD { - err := fmt.Errorf("/sys is not a shared mount on the LXD container, this might be resolved by updating LXD on the host to version 5.0.2 or newer") - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - err = fmt.Errorf("/sys is not a shared mount") - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - } - - if _, err := m.Apply(ctx, ChartCilium, helm.StatePresent, values); err != nil { - err = fmt.Errorf("failed to enable network: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: CiliumAgentImageTag, - Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), - }, err - } - - return types.FeatureStatus{ - Enabled: true, - Version: CiliumAgentImageTag, - Message: EnabledMsg, - }, nil -} - -func rolloutRestartCilium(ctx context.Context, snap snap.Snap, attempts int) error { - client, err := snap.KubernetesClient("") - if err != nil { - return fmt.Errorf("failed to create kubernetes client: %w", err) - } - - if err := control.RetryFor(ctx, attempts, 0, func() error { - if err := client.RestartDeployment(ctx, "cilium-operator", "kube-system"); err != nil { - return fmt.Errorf("failed to restart cilium-operator deployment: %w", err) - } - return nil - }); err != nil { - return fmt.Errorf("failed to restart cilium-operator deployment after %d attempts: %w", attempts, err) - } - - if err := control.RetryFor(ctx, attempts, 0, func() error { - if err := client.RestartDaemonset(ctx, "cilium", "kube-system"); err != nil { - return fmt.Errorf("failed to restart cilium daemonset: %w", err) - } - return nil - }); err != nil { - return fmt.Errorf("failed to restart cilium daemonset after %d attempts: %w", attempts, err) - } - - return nil -} diff --git a/src/k8s/pkg/k8sd/features/cilium/internal.go b/src/k8s/pkg/k8sd/features/cilium/network/internal.go similarity index 99% rename from src/k8s/pkg/k8sd/features/cilium/internal.go rename to src/k8s/pkg/k8sd/features/cilium/network/internal.go index 04ecc1a89..d83895111 100644 --- a/src/k8s/pkg/k8sd/features/cilium/internal.go +++ b/src/k8s/pkg/k8sd/features/cilium/network/internal.go @@ -1,4 +1,4 @@ -package cilium +package network import ( "fmt" diff --git a/src/k8s/pkg/k8sd/features/cilium/internal_test.go b/src/k8s/pkg/k8sd/features/cilium/network/internal_test.go similarity index 99% rename from src/k8s/pkg/k8sd/features/cilium/internal_test.go rename to src/k8s/pkg/k8sd/features/cilium/network/internal_test.go index 44689dc64..1a44fe3f1 100644 --- a/src/k8s/pkg/k8sd/features/cilium/internal_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/network/internal_test.go @@ -1,4 +1,4 @@ -package cilium +package network import ( "testing" diff --git a/src/k8s/pkg/k8sd/features/cilium/network/manifest.go b/src/k8s/pkg/k8sd/features/cilium/network/manifest.go new file mode 100644 index 000000000..417516a8f --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/network/manifest.go @@ -0,0 +1,41 @@ +package network + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + CiliumChartName = "cilium" + + CiliumAgentImageName = "cilium-agent" + CiliumOperatorImageName = "cilium-operator" +) + +var manifest = types.FeatureManifest{ + Name: "network", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + CiliumChartName: { + Name: "cilium", + Version: "1.16.7", + InstallName: "ck-network", + InstallNamespace: "kube-system", + }, + }, + + Images: map[string]types.Image{ + "cilium-agent": { + Registry: "ghcr.io/canonical", + Repository: "cilium", + Tag: "1.16.7-ck0", + }, + "cilium-operator": { + Registry: "ghcr.io/canonical", + Repository: "cilium-operator", + Tag: "1.16.7-ck0", + }, + }, +} + +var FeatureNetwork types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/cilium/network/network.go b/src/k8s/pkg/k8sd/features/cilium/network/network.go new file mode 100644 index 000000000..c4dbf136d --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/network/network.go @@ -0,0 +1,150 @@ +package network + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/cilium" + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/log" + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils" + "github.com/canonical/microcluster/v2/state" +) + +const ( + NetworkDeleteFailedMsgTmpl = "Failed to delete Cilium Network, the error was: %v" + NetworkDeployFailedMsgTmpl = "Failed to deploy Cilium Network, the error was: %v" +) + +// required for unittests. +var ( + GetMountPath = utils.GetMountPath + GetMountPropagationType = utils.GetMountPropagationType +) + +// ApplyNetwork will deploy Cilium when network.Enabled is true. +// ApplyNetwork will remove Cilium when network.Enabled is false. +// ApplyNetwork requires that bpf and cgroups2 are already mounted and available when running under strict snap confinement. If they are not, it will fail (since Cilium will not have the required permissions to mount them). +// ApplyNetwork requires that `/sys` is mounted as a shared mount when running under classic snap confinement. This is to ensure that Cilium will be able to automatically mount bpf and cgroups2 on the pods. +// ApplyNetwork will always return a FeatureStatus indicating the current status of the +// deployment. +// ApplyNetwork returns an error if anything fails. The error is also wrapped in the .Message field of the +// returned FeatureStatus. +func ApplyNetwork(ctx context.Context, snap snap.Snap, m helm.Client, s state.State, apiserver types.APIServer, network types.Network, annotations types.Annotations) (types.FeatureStatus, error) { + ciliumAgentImage := FeatureNetwork.GetImage(CiliumAgentImageName) + + if !network.GetEnabled() { + if _, err := m.Apply(ctx, FeatureNetwork.GetChart(CiliumChartName), helm.StateDeleted, nil); err != nil { + err = fmt.Errorf("failed to uninstall network: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeleteFailedMsgTmpl, err), + }, err + } + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: cilium.DisabledMsg, + }, nil + } + + var values Values = map[string]any{} + + if err := values.ApplyDefaultValues(); err != nil { + err = fmt.Errorf("failed to apply default values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyImageOverrides(); err != nil { + err = fmt.Errorf("failed to calculate image overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), + }, err + } + + if snap.Strict() { + if err := values.ApplyStrictOverrides(); err != nil { + err = fmt.Errorf("failed to calculate strict overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), + }, err + } + } + + if err := values.ApplyClusterConfiguration(ctx, s, apiserver, network); err != nil { + err = fmt.Errorf("failed to calculate cluster config values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyAnnotations(annotations); err != nil { + err = fmt.Errorf("failed to calculate annotation overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), + }, err + } + + if !snap.Strict() { + if err := VerifyMountPropagation(ctx, snap); err != nil { + err = fmt.Errorf("failed to check mount propagation: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), + }, err + } + } + + if _, err := m.Apply(ctx, FeatureNetwork.GetChart(CiliumChartName), helm.StatePresent, values); err != nil { + err = fmt.Errorf("failed to enable network: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: ciliumAgentImage.Tag, + Message: fmt.Sprintf(NetworkDeployFailedMsgTmpl, err), + }, err + } + + return types.FeatureStatus{ + Enabled: true, + Version: ciliumAgentImage.Tag, + Message: cilium.EnabledMsg, + }, nil +} + +func VerifyMountPropagation(ctx context.Context, snap snap.Snap) error { + pt, err := GetMountPropagationType("/sys") + if err != nil { + return fmt.Errorf("failed to get mount propagation type for /sys: %w", err) + } + + if pt == utils.MountPropagationPrivate { + onLXD, err := snap.OnLXD(ctx) + if err != nil { + logger := log.FromContext(ctx) + logger.Error(err, "Failed to check if running on LXD") + } + if onLXD { + return fmt.Errorf("/sys is not a shared mount on the LXD container, this might be resolved by updating LXD on the host to version 5.0.2 or newer") + } + + return fmt.Errorf("/sys is not a shared mount") + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/cilium/network_test.go b/src/k8s/pkg/k8sd/features/cilium/network/network_test.go similarity index 73% rename from src/k8s/pkg/k8sd/features/cilium/network_test.go rename to src/k8s/pkg/k8sd/features/cilium/network/network_test.go index 2566ba6a6..4d21373f7 100644 --- a/src/k8s/pkg/k8sd/features/cilium/network_test.go +++ b/src/k8s/pkg/k8sd/features/cilium/network/network_test.go @@ -1,4 +1,4 @@ -package cilium_test +package network_test import ( "context" @@ -11,6 +11,7 @@ import ( "github.com/canonical/k8s/pkg/client/helm/loader" helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" snapmock "github.com/canonical/k8s/pkg/snap/mock" @@ -52,16 +53,16 @@ func TestNetworkDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.NetworkDeleteFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_network.NetworkDeleteFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -83,16 +84,16 @@ func TestNetworkDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(Equal(cilium.DisabledMsg)) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -119,11 +120,11 @@ func TestNetworkEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) }) @@ -146,16 +147,16 @@ func TestNetworkEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, annotations) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, annotations) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) g.Expect(status.Message).To(Equal(cilium.EnabledMsg)) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateNetworkValues(g, callArgs.Values, network, snapM) }) @@ -181,16 +182,16 @@ func TestNetworkEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, annotations) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, annotations) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.NetworkDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_network.NetworkDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateNetworkValues(g, callArgs.Values, network, snapM) }) @@ -218,16 +219,16 @@ func TestNetworkEnabled(t *testing.T) { apiv1_annotations.AnnotationCNIExclusive: "true", } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, testAnnotations) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, testAnnotations) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) g.Expect(status.Message).To(Equal(cilium.EnabledMsg)) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) cniValues := callArgs.Values["cni"].(map[string]interface{}) @@ -258,16 +259,16 @@ func TestNetworkEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, testAnnotations) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, testAnnotations) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) g.Expect(status.Message).To(Equal(cilium.EnabledMsg)) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(cilium.ChartCilium)) + g.Expect(callArgs.Chart).To(Equal(cilium_network.FeatureNetwork.GetChart(cilium_network.CiliumChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) sctpValues := callArgs.Values["sctp"].(map[string]interface{}) @@ -302,7 +303,8 @@ func TestNetworkMountPath(t *testing.T) { apiserver := types.APIServer{ SecurePort: ptr.To(6443), } - cilium.GetMountPath = func(fsType string) (string, error) { + + cilium_network.GetMountPath = func(fsType string) (string, error) { if fsType == tc.name { return "", mountPathErr } @@ -310,13 +312,13 @@ func TestNetworkMountPath(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(mountPathErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.NetworkDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_network.NetworkDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) }) } @@ -329,7 +331,7 @@ func TestNetworkMountPropagationType(t *testing.T) { g := NewWithT(t) mountErr := errors.New("/sys not found") - cilium.GetMountPropagationType = func(path string) (utils.MountPropagationType, error) { + cilium_network.GetMountPropagationType = func(path string) (utils.MountPropagationType, error) { return "", mountErr } helmM := &helmmock.Mock{} @@ -348,21 +350,21 @@ func TestNetworkMountPropagationType(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(mountErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.NetworkDeployFailedMsgTmpl, err))) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_network.NetworkDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) }) t.Run("MountPropagationPrivateOnLXDError", func(t *testing.T) { g := NewWithT(t) - cilium.GetMountPropagationType = func(path string) (utils.MountPropagationType, error) { + cilium_network.GetMountPropagationType = func(path string) (utils.MountPropagationType, error) { return utils.MountPropagationPrivate, nil } helmM := &helmmock.Mock{} @@ -384,13 +386,13 @@ func TestNetworkMountPropagationType(t *testing.T) { ctx := klog.NewContext(context.Background(), logger) mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(ctx, snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(ctx, snapM, mc, s, apiserver, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.NetworkDeployFailedMsgTmpl, err))) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_network.NetworkDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) testingLogger, ok := logger.GetSink().(ktesting.Underlier) if !ok { @@ -402,7 +404,7 @@ func TestNetworkMountPropagationType(t *testing.T) { t.Run("MountPropagationPrivateOnLXD", func(t *testing.T) { g := NewWithT(t) - cilium.GetMountPropagationType = func(path string) (utils.MountPropagationType, error) { + cilium_network.GetMountPropagationType = func(path string) (utils.MountPropagationType, error) { return utils.MountPropagationPrivate, nil } helmM := &helmmock.Mock{} @@ -422,20 +424,20 @@ func TestNetworkMountPropagationType(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.NetworkDeployFailedMsgTmpl, err))) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_network.NetworkDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) }) t.Run("MountPropagationPrivate", func(t *testing.T) { g := NewWithT(t) - cilium.GetMountPropagationType = func(_ string) (utils.MountPropagationType, error) { + cilium_network.GetMountPropagationType = func(_ string) (utils.MountPropagationType, error) { return utils.MountPropagationPrivate, nil } helmM := &helmmock.Mock{} @@ -454,13 +456,13 @@ func TestNetworkMountPropagationType(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&cilium.ChartFS)) - status, err := cilium.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) + status, err := cilium_network.ApplyNetwork(context.Background(), snapM, mc, s, apiserver, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium.NetworkDeployFailedMsgTmpl, err))) + g.Expect(status.Message).To(Equal(fmt.Sprintf(cilium_network.NetworkDeployFailedMsgTmpl, err))) - g.Expect(status.Version).To(Equal(cilium.CiliumAgentImageTag)) + g.Expect(status.Version).To(Equal(cilium_network.FeatureNetwork.GetImage(cilium_network.CiliumAgentImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(BeEmpty()) }) }) diff --git a/src/k8s/pkg/k8sd/features/cilium/network/register.go b/src/k8s/pkg/k8sd/features/cilium/network/register.go new file mode 100644 index 000000000..399c88997 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/network/register.go @@ -0,0 +1,17 @@ +package network + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + ciliumAgentImage := FeatureNetwork.GetImage(CiliumAgentImageName) + ciliumOperatorImage := FeatureNetwork.GetImage(CiliumOperatorImageName) + + images.Register( + fmt.Sprintf("%s:%s", ciliumAgentImage.GetURI(), ciliumAgentImage.Tag), + fmt.Sprintf("%s-generic:%s", ciliumOperatorImage.GetURI(), ciliumOperatorImage.Tag), + ) +} diff --git a/src/k8s/pkg/k8sd/features/cilium/network/values.go b/src/k8s/pkg/k8sd/features/cilium/network/values.go new file mode 100644 index 000000000..690f9e11e --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/network/values.go @@ -0,0 +1,216 @@ +package network + +import ( + "context" + "fmt" + "net" + "strings" + + "dario.cat/mergo" + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/utils" + "github.com/canonical/microcluster/v2/state" +) + +type Values map[string]any + +func (v Values) ApplyDefaultValues() error { + values := map[string]any{ + "image": map[string]any{ + "useDigest": false, + }, + "socketLB": map[string]any{ + "enabled": true, + }, + "cni": map[string]any{ + "confPath": "/etc/cni/net.d", + "binPath": "/opt/cni/bin", + }, + "operator": map[string]any{ + "replicas": 1, + "image": map[string]any{ + "useDigest": false, + }, + }, + "envoy": map[string]any{ + "enabled": false, // 1.16+ installs envoy as a standalone daemonset by default if not explicitly disabled + }, + // https://docs.cilium.io/en/v1.15/network/kubernetes/kubeproxy-free/#kube-proxy-hybrid-modes + "nodePort": map[string]any{ + "enabled": true, + "enableHealthCheck": false, + }, + "disableEnvoyVersionCheck": true, + // This flag enables the runtime device detection which is set to true by default in Cilium 1.16+ + "enableRuntimeDeviceDetection": true, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride, mergo.WithTypeCheck); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyImageOverrides() error { + ciliumAgentImage := FeatureNetwork.GetImage(CiliumAgentImageName) + ciliumOperatorImage := FeatureNetwork.GetImage(CiliumOperatorImageName) + + values := map[string]any{ + "image": map[string]any{ + "repository": ciliumAgentImage.GetURI(), + "tag": ciliumAgentImage.Tag, + }, + + "operator": map[string]any{ + "image": map[string]any{ + "repository": ciliumOperatorImage.GetURI(), + "tag": ciliumOperatorImage.Tag, + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +func (v Values) ApplyClusterConfiguration(ctx context.Context, s state.State, apiserver types.APIServer, network types.Network) error { + c, err := s.Leader() + if err != nil { + return fmt.Errorf("failed to get leader client: %w", err) + } + + clusterMembers, err := c.GetClusterMembers(ctx) + if err != nil { + return fmt.Errorf("failed to get cluster members: %w", err) + } + + localhostAddress, err := utils.DetermineLocalhostAddress(clusterMembers) + if err != nil { + return fmt.Errorf("failed to determine localhost address: %w", err) + } + + nodeIP := net.ParseIP(s.Address().Hostname()) + if nodeIP == nil { + return fmt.Errorf("failed to parse node IP address %q", s.Address().Hostname()) + } + + defaultCidr, err := utils.FindCIDRForIP(nodeIP) + if err != nil { + return fmt.Errorf("failed to find cidr of default interface: %w", err) + } + + ipv4CIDR, ipv6CIDR, err := utils.SplitCIDRStrings(network.GetPodCIDR()) + if err != nil { + return fmt.Errorf("invalid kube-proxy --cluster-cidr value: %w", err) + } + + values := map[string]any{ + "ipv4": map[string]any{ + "enabled": ipv4CIDR != "", + }, + "ipv6": map[string]any{ + "enabled": ipv6CIDR != "", + }, + "ipam": map[string]any{ + "operator": map[string]any{ + "clusterPoolIPv4PodCIDRList": ipv4CIDR, + "clusterPoolIPv6PodCIDRList": ipv6CIDR, + }, + }, + // socketLB requires an endpoint to the apiserver that's not managed by the kube-proxy + // so we point to the localhost:secureport to talk to either the kube-apiserver or the kube-apiserver-proxy + "k8sServiceHost": strings.Trim(localhostAddress, "[]"), // Cilium already adds the brackets for ipv6 addresses, so we need to remove them + "k8sServicePort": apiserver.GetSecurePort(), + } + + // If we are deploying with IPv6 only, we need to set the routing mode to native + if ipv4CIDR == "" && ipv6CIDR != "" { + values["routingMode"] = "native" + values["ipv6NativeRoutingCIDR"] = defaultCidr + values["autoDirectNodeRoutes"] = true + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyStrictOverrides() error { + bpfMnt, err := GetMountPath("bpf") + if err != nil { + return fmt.Errorf("failed to get bpf mount path: %w", err) + } + + cgrMnt, err := GetMountPath("cgroup2") + if err != nil { + return fmt.Errorf("failed to get cgroup2 mount path: %w", err) + } + + values := map[string]any{ + "bpf": map[string]any{ + "autoMount": map[string]any{ + "enabled": false, + }, + "root": bpfMnt, + }, + "cgroup": map[string]any{ + "autoMount": map[string]any{ + "enabled": false, + }, + "hostRoot": cgrMnt, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge strict overrides values: %w", err) + } + + return nil +} + +func (v Values) ApplyAnnotations(annotations types.Annotations) error { + config, err := internalConfig(annotations) + if err != nil { + return fmt.Errorf("failed to parse annotations: %w", err) + } + + ciliumNodePortValues := map[string]any{} + + if config.directRoutingDevice != "" { + ciliumNodePortValues["directRoutingDevice"] = config.directRoutingDevice + } + + bpfValues := map[string]any{} + if config.vlanBPFBypass != nil { + bpfValues["vlanBypass"] = config.vlanBPFBypass + } + + values := map[string]any{ + "bpf": bpfValues, + + "cni": map[string]any{ + "exclusive": config.cniExclusive, + }, + "sctp": map[string]any{ + "enabled": config.sctpEnabled, + }, + + "nodePort": ciliumNodePortValues, + } + + if config.devices != "" { + values["devices"] = config.devices + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge strict overrides values: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/cilium/register.go b/src/k8s/pkg/k8sd/features/cilium/register.go index 76b2f806b..7a9c56eff 100644 --- a/src/k8s/pkg/k8sd/features/cilium/register.go +++ b/src/k8s/pkg/k8sd/features/cilium/register.go @@ -1,17 +1,9 @@ package cilium import ( - "fmt" - "github.com/canonical/k8s/pkg/k8sd/charts" - "github.com/canonical/k8s/pkg/k8sd/images" ) func init() { - images.Register( - fmt.Sprintf("%s:%s", ciliumAgentImageRepo, CiliumAgentImageTag), - fmt.Sprintf("%s-generic:%s", ciliumOperatorImageRepo, ciliumOperatorImageTag), - ) - charts.Register(&ChartFS) } diff --git a/src/k8s/pkg/k8sd/features/cilium/util.go b/src/k8s/pkg/k8sd/features/cilium/util.go new file mode 100644 index 000000000..d23e7d26e --- /dev/null +++ b/src/k8s/pkg/k8sd/features/cilium/util.go @@ -0,0 +1,36 @@ +package cilium + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils/control" +) + +func RolloutRestartCilium(ctx context.Context, snap snap.Snap, attempts int) error { + client, err := snap.KubernetesClient("") + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + if err := control.RetryFor(ctx, attempts, 0, func() error { + if err := client.RestartDeployment(ctx, "cilium-operator", "kube-system"); err != nil { + return fmt.Errorf("failed to restart cilium-operator deployment: %w", err) + } + return nil + }); err != nil { + return fmt.Errorf("failed to restart cilium-operator deployment after %d attempts: %w", attempts, err) + } + + if err := control.RetryFor(ctx, attempts, 0, func() error { + if err := client.RestartDaemonset(ctx, "cilium", "kube-system"); err != nil { + return fmt.Errorf("failed to restart cilium daemonset: %w", err) + } + return nil + }); err != nil { + return fmt.Errorf("failed to restart cilium daemonset after %d attempts: %w", attempts, err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/contour/chart.go b/src/k8s/pkg/k8sd/features/contour/chart.go index b08371815..81e6ff777 100644 --- a/src/k8s/pkg/k8sd/features/contour/chart.go +++ b/src/k8s/pkg/k8sd/features/contour/chart.go @@ -2,68 +2,7 @@ package contour import ( "embed" - - "github.com/canonical/k8s/pkg/client/helm" ) //go:embed all:charts var ChartFS embed.FS - -var ( - // chartContour represents manifests to deploy Contour. - // This excludes shared CRDs. - chartContour = helm.InstallableChart{ - Name: "contour", - Version: "17.0.4", - InstallName: "ck-ingress", - InstallNamespace: "projectcontour", - } - // chartGateway represents manifests to deploy Contour Gateway. - // This excludes shared CRDs. - chartGateway = helm.InstallableChart{ - Name: "ck-gateway-contour", - Version: "1.28.2", - InstallName: "ck-gateway", - InstallNamespace: "projectcontour", - } - // chartDefaultTLS represents manifests to deploy a delegation resource for the default TLS secret. - chartDefaultTLS = helm.InstallableChart{ - Name: "ck-ingress-tls", - Version: "0.1.0", - InstallName: "ck-ingress-tls", - InstallNamespace: "projectcontour-root", - } - // chartCommonContourCRDS represents manifests to deploy common Contour CRDs. - chartCommonContourCRDS = helm.InstallableChart{ - Name: "ck-contour-common", - Version: "1.28.2", - InstallName: "ck-contour-common", - InstallNamespace: "projectcontour", - } - - // ContourGatewayProvisionerEnvoyImageRepo represents the image to use for envoy in the gateway. - ContourGatewayProvisionerEnvoyImageRepo = "ghcr.io/canonical/k8s-snap/envoyproxy/envoy" - - // NOTE: The image version is v1.29.2 instead of 1.28.2 - // to follow the upstream configuration for the contour gateway provisioner. - // ContourGatewayProvisionerEnvoyImageTag is the tag to use for envoy in the gateway. - ContourGatewayProvisionerEnvoyImageTag = "v1.29.2" - - // ContourIngressEnvoyImageRepo represents the image to use for the Contour Envoy proxy. - ContourIngressEnvoyImageRepo = "ghcr.io/canonical/k8s-snap/bitnami/envoy" - - // ContourIngressEnvoyImageTag is the tag to use for the Contour Envoy proxy image. - ContourIngressEnvoyImageTag = "1.28.2-debian-12-r0" - - // ContourIngressContourImageRepo represents the image to use for Contour. - ContourIngressContourImageRepo = "ghcr.io/canonical/k8s-snap/bitnami/contour" - - // ContourIngressContourImageTag is the tag to use for the Contour image. - ContourIngressContourImageTag = "1.28.2-debian-12-r4" - - // ContourGatewayProvisionerContourImageRepo represents the image to use for the Contour Gateway Provisioner. - ContourGatewayProvisionerContourImageRepo = "ghcr.io/canonical/k8s-snap/projectcontour/contour" - - // ContourGatewayProvisionerContourImageTag is the tag to use for the Contour Gateway Provisioner image. - ContourGatewayProvisionerContourImageTag = "v1.28.2" -) diff --git a/src/k8s/pkg/k8sd/features/contour/gateway.go b/src/k8s/pkg/k8sd/features/contour/gateway.go deleted file mode 100644 index f06e3367d..000000000 --- a/src/k8s/pkg/k8sd/features/contour/gateway.go +++ /dev/null @@ -1,136 +0,0 @@ -package contour - -import ( - "context" - "fmt" - - "github.com/canonical/k8s/pkg/client/helm" - "github.com/canonical/k8s/pkg/k8sd/types" - "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils/control" -) - -const ( - EnabledMsg = "enabled" - DisabledMsg = "disabled" - GatewayDeployFailedMsgTmpl = "Failed to deploy Contour Gateway, the error was: %v" - GatewayDeleteFailedMsgTmpl = "Failed to delete Contour Gateway, the error was: %v" -) - -// ApplyGateway will install a helm chart for contour-gateway-provisioner on the cluster when gateway.Enabled is true. -// ApplyGateway will uninstall the helm chart for contour-gateway-provisioner from the cluster when gateway.Enabled is false. -// ApplyGateway will apply common contour CRDS, these are shared with ingress. -// ApplyGateway will always return a FeatureStatus indicating the current status of the -// deployment. -// ApplyGateway returns an error if anything fails. The error is also wrapped in the .Message field of the -// returned FeatureStatus. -func ApplyGateway(ctx context.Context, snap snap.Snap, m helm.Client, gateway types.Gateway, network types.Network, _ types.Annotations) (types.FeatureStatus, error) { - if !gateway.GetEnabled() { - if _, err := m.Apply(ctx, chartGateway, helm.StateDeleted, nil); err != nil { - err = fmt.Errorf("failed to uninstall the contour gateway chart: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: ContourGatewayProvisionerContourImageTag, - Message: fmt.Sprintf(GatewayDeleteFailedMsgTmpl, err), - }, err - } - return types.FeatureStatus{ - Enabled: false, - Version: ContourGatewayProvisionerContourImageTag, - Message: DisabledMsg, - }, nil - } - - // Apply common contour CRDS, these are shared with ingress - if err := applyCommonContourCRDS(ctx, snap, m, true); err != nil { - err = fmt.Errorf("failed to apply common contour CRDS: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: ContourGatewayProvisionerContourImageTag, - Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), - }, err - } - - if err := waitForRequiredContourCommonCRDs(ctx, snap); err != nil { - err = fmt.Errorf("failed to wait for required contour common CRDs to be available: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: ContourGatewayProvisionerContourImageTag, - Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), - }, err - } - - values := map[string]any{ - "projectcontour": map[string]any{ - "image": map[string]any{ - "repository": ContourGatewayProvisionerContourImageRepo, - "tag": ContourGatewayProvisionerContourImageTag, - }, - }, - "envoyproxy": map[string]any{ - "image": map[string]any{ - "repository": ContourGatewayProvisionerEnvoyImageRepo, - "tag": ContourGatewayProvisionerEnvoyImageTag, - }, - }, - } - - if _, err := m.Apply(ctx, chartGateway, helm.StatePresent, values); err != nil { - err = fmt.Errorf("failed to install the contour gateway chart: %w", err) - return types.FeatureStatus{ - Enabled: false, - Version: ContourGatewayProvisionerContourImageTag, - Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), - }, err - } - - return types.FeatureStatus{ - Enabled: true, - Version: ContourGatewayProvisionerContourImageTag, - Message: EnabledMsg, - }, nil -} - -// waitForRequiredContourCommonCRDs waits for the required contour CRDs to be available -// by checking the API resources by group version. -func waitForRequiredContourCommonCRDs(ctx context.Context, snap snap.Snap) error { - client, err := snap.KubernetesClient("") - if err != nil { - return fmt.Errorf("failed to create Kubernetes client: %w", err) - } - - return control.WaitUntilReady(ctx, func() (bool, error) { - resourcesV1Alpha, err := client.ListResourcesForGroupVersion("projectcontour.io/v1alpha1") - if err != nil { - // This error is expected if the group version is not yet deployed. - return false, nil - } - resourcesV1, err := client.ListResourcesForGroupVersion("projectcontour.io/v1") - if err != nil { - // This error is expected if the group version is not yet deployed. - return false, nil - } - - requiredCRDs := map[string]bool{ - "projectcontour.io/v1alpha1:contourconfigurations": true, - "projectcontour.io/v1alpha1:contourdeployments": true, - "projectcontour.io/v1alpha1:extensionservices": true, - "projectcontour.io/v1:tlscertificatedelegations": true, - "projectcontour.io/v1:httpproxies": true, - } - - requiredCount := len(requiredCRDs) - for _, resource := range resourcesV1Alpha.APIResources { - if _, exists := requiredCRDs[fmt.Sprintf("projectcontour.io/v1alpha1:%s", resource.Name)]; exists { - requiredCount-- - } - } - for _, resource := range resourcesV1.APIResources { - if _, exists := requiredCRDs[fmt.Sprintf("projectcontour.io/v1:%s", resource.Name)]; exists { - requiredCount-- - } - } - - return requiredCount == 0, nil - }) -} diff --git a/src/k8s/pkg/k8sd/features/contour/gateway/gateway.go b/src/k8s/pkg/k8sd/features/contour/gateway/gateway.go new file mode 100644 index 000000000..a56b51c33 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/gateway/gateway.go @@ -0,0 +1,89 @@ +package gateway + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/contour" + contour_ingress "github.com/canonical/k8s/pkg/k8sd/features/contour/ingress" + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/snap" +) + +const ( + GatewayDeployFailedMsgTmpl = "Failed to deploy Contour Gateway, the error was: %v" + GatewayDeleteFailedMsgTmpl = "Failed to delete Contour Gateway, the error was: %v" +) + +// ApplyGateway will install a helm chart for contour-gateway-provisioner on the cluster when gateway.Enabled is true. +// ApplyGateway will uninstall the helm chart for contour-gateway-provisioner from the cluster when gateway.Enabled is false. +// ApplyGateway will apply common contour CRDS, these are shared with ingress. +// ApplyGateway will always return a FeatureStatus indicating the current status of the +// deployment. +// ApplyGateway returns an error if anything fails. The error is also wrapped in the .Message field of the +// returned FeatureStatus. +func ApplyGateway(ctx context.Context, snap snap.Snap, m helm.Client, gateway types.Gateway, network types.Network, _ types.Annotations) (types.FeatureStatus, error) { + contourGatewayProvisionerContourImage := FeatureGateway.GetImage(ContourGatewayProvisionerContourImageName) + + if !gateway.GetEnabled() { + if _, err := m.Apply(ctx, FeatureGateway.GetChart(ChartGatewayName), helm.StateDeleted, nil); err != nil { + err = fmt.Errorf("failed to uninstall the contour gateway chart: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourGatewayProvisionerContourImage.Tag, + Message: fmt.Sprintf(GatewayDeleteFailedMsgTmpl, err), + }, err + } + return types.FeatureStatus{ + Enabled: false, + Version: contourGatewayProvisionerContourImage.Tag, + Message: contour.DisabledMsg, + }, nil + } + + // Apply common contour CRDS, these are shared with ingress + if err := contour_ingress.ApplyCommonContourCRDS(ctx, snap, m, true); err != nil { + err = fmt.Errorf("failed to apply common contour CRDS: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourGatewayProvisionerContourImage.Tag, + Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), + }, err + } + + if err := contour.WaitForRequiredContourCommonCRDs(ctx, snap); err != nil { + err = fmt.Errorf("failed to wait for required contour common CRDs to be available: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourGatewayProvisionerContourImage.Tag, + Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), + }, err + } + + var values Values = map[string]any{} + + if err := values.ApplyImageOverrides(); err != nil { + err = fmt.Errorf("failed to apply image overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourGatewayProvisionerContourImage.Tag, + Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), + }, err + } + + if _, err := m.Apply(ctx, FeatureGateway.GetChart(ChartGatewayName), helm.StatePresent, values); err != nil { + err = fmt.Errorf("failed to install the contour gateway chart: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourGatewayProvisionerContourImage.Tag, + Message: fmt.Sprintf(GatewayDeployFailedMsgTmpl, err), + }, err + } + + return types.FeatureStatus{ + Enabled: true, + Version: contourGatewayProvisionerContourImage.Tag, + Message: contour.EnabledMsg, + }, nil +} diff --git a/src/k8s/pkg/k8sd/features/contour/gateway_test.go b/src/k8s/pkg/k8sd/features/contour/gateway/gateway_test.go similarity index 69% rename from src/k8s/pkg/k8sd/features/contour/gateway_test.go rename to src/k8s/pkg/k8sd/features/contour/gateway/gateway_test.go index 174d2f0fd..bace49775 100644 --- a/src/k8s/pkg/k8sd/features/contour/gateway_test.go +++ b/src/k8s/pkg/k8sd/features/contour/gateway/gateway_test.go @@ -1,4 +1,4 @@ -package contour_test +package gateway_test import ( "context" @@ -11,6 +11,7 @@ import ( helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/client/kubernetes" "github.com/canonical/k8s/pkg/k8sd/features/contour" + contour_gateway "github.com/canonical/k8s/pkg/k8sd/features/contour/gateway" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -39,13 +40,13 @@ func TestGatewayDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := contour_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring(applyErr.Error())) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourGatewayProvisionerContourImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(contour.GatewayDeleteFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerContourImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(contour_gateway.GatewayDeleteFailedMsgTmpl, err))) }) t.Run("Success", func(t *testing.T) { @@ -63,11 +64,11 @@ func TestGatewayDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := contour_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourGatewayProvisionerContourImageTag)) + g.Expect(status.Version).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerContourImageName).Tag)) g.Expect(status.Message).To(Equal(contour.DisabledMsg)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -92,13 +93,13 @@ func TestGatewayEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := contour_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourGatewayProvisionerContourImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(contour.GatewayDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerContourImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(contour_gateway.GatewayDeployFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -142,11 +143,11 @@ func TestGatewayEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) + status, err := contour_gateway.ApplyGateway(context.Background(), snapM, mc, gateway, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(contour.ContourGatewayProvisionerContourImageTag)) + g.Expect(status.Version).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerContourImageName).Tag)) g.Expect(status.Message).To(Equal(contour.EnabledMsg)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(2)) @@ -155,14 +156,14 @@ func TestGatewayEnabled(t *testing.T) { g.Expect(ok).To(BeTrue()) contourImage, ok := contourValues["image"].(map[string]any) g.Expect(ok).To(BeTrue()) - g.Expect(contourImage["repository"]).To(Equal(contour.ContourGatewayProvisionerContourImageRepo)) - g.Expect(contourImage["tag"]).To(Equal(contour.ContourGatewayProvisionerContourImageTag)) + g.Expect(contourImage["repository"]).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerContourImageName).GetURI())) + g.Expect(contourImage["tag"]).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerContourImageName).Tag)) envoyValues, ok := values["envoyproxy"].(map[string]any) g.Expect(ok).To(BeTrue()) envoyImage, ok := envoyValues["image"].(map[string]any) g.Expect(ok).To(BeTrue()) - g.Expect(envoyImage["repository"]).To(Equal(contour.ContourGatewayProvisionerEnvoyImageRepo)) - g.Expect(envoyImage["tag"]).To(Equal(contour.ContourGatewayProvisionerEnvoyImageTag)) + g.Expect(envoyImage["repository"]).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerEnvoyImageName).GetURI())) + g.Expect(envoyImage["tag"]).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerEnvoyImageName).Tag)) }) t.Run("CrdDeploymentFailed", func(t *testing.T) { @@ -204,13 +205,13 @@ func TestGatewayEnabled(t *testing.T) { defer cancel() mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyGateway(ctx, snapM, mc, gateway, network, nil) + status, err := contour_gateway.ApplyGateway(ctx, snapM, mc, gateway, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("failed to wait for required contour common CRDs")) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourGatewayProvisionerContourImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(contour.GatewayDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(contour_gateway.FeatureGateway.GetImage(contour_gateway.ContourGatewayProvisionerContourImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(contour_gateway.GatewayDeployFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) } diff --git a/src/k8s/pkg/k8sd/features/contour/gateway/manifest.go b/src/k8s/pkg/k8sd/features/contour/gateway/manifest.go new file mode 100644 index 000000000..a25df7344 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/gateway/manifest.go @@ -0,0 +1,41 @@ +package gateway + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + ChartGatewayName = "ck-gateway-contour" + + ContourGatewayProvisionerEnvoyImageName = "envoy-gateway" + ContourGatewayProvisionerContourImageName = "contour-gateway" +) + +var manifest = types.FeatureManifest{ + Name: "gateway", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + ChartGatewayName: { + Name: "ck-gateway-contour", + Version: "1.28.2", + InstallName: "ck-gateway", + InstallNamespace: "projectcontour", + }, + }, + + Images: map[string]types.Image{ + ContourGatewayProvisionerContourImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "projectcontour/contour", + Tag: "v1.28.2", + }, + ContourGatewayProvisionerEnvoyImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "envoyproxy/envoy", + Tag: "v1.29.2", + }, + }, +} + +var FeatureGateway types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/contour/gateway/register.go b/src/k8s/pkg/k8sd/features/contour/gateway/register.go new file mode 100644 index 000000000..d5f60402a --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/gateway/register.go @@ -0,0 +1,17 @@ +package gateway + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + contourGatewayProvisionerContourImage := FeatureGateway.GetImage(ContourGatewayProvisionerContourImageName) + contourGatewayProvisionerEnvoyImage := FeatureGateway.GetImage(ContourGatewayProvisionerEnvoyImageName) + + images.Register( + fmt.Sprintf("%s:%s", contourGatewayProvisionerContourImage.GetURI(), contourGatewayProvisionerContourImage.Tag), + fmt.Sprintf("%s:%s", contourGatewayProvisionerEnvoyImage.GetURI(), contourGatewayProvisionerEnvoyImage.Tag), + ) +} diff --git a/src/k8s/pkg/k8sd/features/contour/gateway/values.go b/src/k8s/pkg/k8sd/features/contour/gateway/values.go new file mode 100644 index 000000000..fdeae56e9 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/gateway/values.go @@ -0,0 +1,35 @@ +package gateway + +import ( + "fmt" + + "dario.cat/mergo" +) + +type Values map[string]any + +func (v Values) ApplyImageOverrides() error { + contourGatewayProvisionerContourImage := FeatureGateway.GetImage(ContourGatewayProvisionerContourImageName) + contourGatewayProvisionerEnvoyImage := FeatureGateway.GetImage(ContourGatewayProvisionerEnvoyImageName) + + values := map[string]any{ + "projectcontour": map[string]any{ + "image": map[string]any{ + "repository": contourGatewayProvisionerContourImage.GetURI(), + "tag": contourGatewayProvisionerContourImage.Tag, + }, + }, + "envoyproxy": map[string]any{ + "image": map[string]any{ + "repository": contourGatewayProvisionerEnvoyImage.GetURI(), + "tag": contourGatewayProvisionerEnvoyImage.Tag, + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/contour/ingress.go b/src/k8s/pkg/k8sd/features/contour/ingress/ingress.go similarity index 54% rename from src/k8s/pkg/k8sd/features/contour/ingress.go rename to src/k8s/pkg/k8sd/features/contour/ingress/ingress.go index 5d2f28882..bd96fa4c9 100644 --- a/src/k8s/pkg/k8sd/features/contour/ingress.go +++ b/src/k8s/pkg/k8sd/features/contour/ingress/ingress.go @@ -1,13 +1,13 @@ -package contour +package ingress import ( "context" "fmt" "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/contour" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" - "github.com/canonical/k8s/pkg/utils/control" ) const ( @@ -26,96 +26,88 @@ const ( // returned FeatureStatus. // Contour CRDS are applied through a ck-contour common chart (Overlap with gateway). func ApplyIngress(ctx context.Context, snap snap.Snap, m helm.Client, ingress types.Ingress, _ types.Network, _ types.Annotations) (types.FeatureStatus, error) { + contourIngressContourImage := FeatureIngress.GetImage(ContourIngressContourImageName) + if !ingress.GetEnabled() { - if _, err := m.Apply(ctx, chartContour, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureIngress.GetChart(ChartContourName), helm.StateDeleted, nil); err != nil { err = fmt.Errorf("failed to uninstall ingress: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, + Version: contourIngressContourImage.Tag, Message: fmt.Sprintf(IngressDeleteFailedMsgTmpl, err), }, err } return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, - Message: DisabledMsg, + Version: contourIngressContourImage.Tag, + Message: contour.DisabledMsg, }, nil } // Apply common contour CRDS, these are shared with gateway - if err := applyCommonContourCRDS(ctx, snap, m, true); err != nil { + if err := ApplyCommonContourCRDS(ctx, snap, m, true); err != nil { err = fmt.Errorf("failed to apply common contour CRDS: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, + Version: contourIngressContourImage.Tag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err } - if err := waitForRequiredContourCommonCRDs(ctx, snap); err != nil { + if err := contour.WaitForRequiredContourCommonCRDs(ctx, snap); err != nil { err = fmt.Errorf("failed to wait for required contour common CRDs to be available: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, + Version: contourIngressContourImage.Tag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err } - var values map[string]any - values = map[string]any{ - "envoy-service-namespace": "projectcontour", - "envoy-service-name": "envoy", - "envoy": map[string]any{ - "image": map[string]any{ - "registry": "", - "repository": ContourIngressEnvoyImageRepo, - "tag": ContourIngressEnvoyImageTag, - }, - }, - "contour": map[string]any{ - "manageCRDs": false, - "ingressClass": map[string]any{ - "name": "ck-ingress", - "create": true, - "default": true, - }, - "image": map[string]any{ - "registry": "", - "repository": ContourIngressContourImageRepo, - "tag": ContourIngressContourImageTag, - }, - }, + var values Values = map[string]any{} + + if err := values.ApplyDefaultValues(); err != nil { + err = fmt.Errorf("failed to apply default values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourIngressContourImage.Tag, + Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), + }, err } - if ingress.GetEnableProxyProtocol() { - contour, ok := values["contour"].(map[string]any) - if !ok { - err := fmt.Errorf("unexpected type for contour values") - return types.FeatureStatus{ - Enabled: false, - Version: ContourIngressContourImageTag, - Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), - }, err - } - contour["extraArgs"] = []string{"--use-proxy-protocol"} + if err := values.ApplyImageOverrides(); err != nil { + err = fmt.Errorf("failed to apply image overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourIngressContourImage.Tag, + Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), + }, err } - changed, err := m.Apply(ctx, chartContour, helm.StatePresent, values) + if err := values.ApplyClusterConfiguration(ingress); err != nil { + err = fmt.Errorf("failed to apply cluster configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourIngressContourImage.Tag, + Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), + }, err + } + + changed, err := m.Apply(ctx, FeatureIngress.GetChart(ChartContourName), helm.StatePresent, values) if err != nil { err = fmt.Errorf("failed to enable ingress: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, + Version: contourIngressContourImage.Tag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err } if changed { - if err := rolloutRestartContour(ctx, snap, 3); err != nil { + if err := contour.RolloutRestartContour(ctx, snap, 3); err != nil { err = fmt.Errorf("failed to rollout restart contour to apply ingress: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, + Version: contourIngressContourImage.Tag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err } @@ -125,29 +117,37 @@ func ApplyIngress(ctx context.Context, snap snap.Snap, m helm.Client, ingress ty // The default TLS secret is created by the user // and gets set via k8s set defaultTLSSecret=bananas. if ingress.GetDefaultTLSSecret() != "" { - values = map[string]any{ - "defaultTLSSecret": ingress.GetDefaultTLSSecret(), + var tlsValues TLSValues = map[string]any{} + + if err := tlsValues.ApplyClusterConfiguration(ingress); err != nil { + err = fmt.Errorf("failed to apply cluster configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: contourIngressContourImage.Tag, + Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), + }, err } - if _, err := m.Apply(ctx, chartDefaultTLS, helm.StatePresent, values); err != nil { + + if _, err := m.Apply(ctx, FeatureIngress.GetChart(ChartDefaultTLSName), helm.StatePresent, tlsValues); err != nil { err = fmt.Errorf("failed to install the delegation resource for default TLS secret: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, + Version: contourIngressContourImage.Tag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err } return types.FeatureStatus{ Enabled: true, - Version: ContourIngressContourImageTag, - Message: EnabledMsg, + Version: contourIngressContourImage.Tag, + Message: contour.EnabledMsg, }, nil } - if _, err := m.Apply(ctx, chartDefaultTLS, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureIngress.GetChart(ChartDefaultTLSName), helm.StateDeleted, nil); err != nil { err = fmt.Errorf("failed to uninstall the delegation resource for default TLS secret: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ContourIngressContourImageTag, + Version: contourIngressContourImage.Tag, Message: fmt.Sprintf(IngressDeployFailedMsgTmpl, err), }, err @@ -155,43 +155,24 @@ func ApplyIngress(ctx context.Context, snap snap.Snap, m helm.Client, ingress ty return types.FeatureStatus{ Enabled: true, - Version: ContourIngressContourImageTag, - Message: EnabledMsg, + Version: contourIngressContourImage.Tag, + Message: contour.EnabledMsg, }, nil } // applyCommonContourCRDS will install the common contour CRDS when enabled is true. // These CRDS are shared between the contour ingress and the gateway feature. -func applyCommonContourCRDS(ctx context.Context, snap snap.Snap, m helm.Client, enabled bool) error { +func ApplyCommonContourCRDS(ctx context.Context, snap snap.Snap, m helm.Client, enabled bool) error { if enabled { - if _, err := m.Apply(ctx, chartCommonContourCRDS, helm.StatePresent, nil); err != nil { + if _, err := m.Apply(ctx, FeatureIngress.GetChart(ChartCommonContourCRDSName), helm.StatePresent, nil); err != nil { return fmt.Errorf("failed to install common CRDS: %w", err) } return nil } - if _, err := m.Apply(ctx, chartCommonContourCRDS, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureIngress.GetChart(ChartCommonContourCRDSName), helm.StateDeleted, nil); err != nil { return fmt.Errorf("failed to uninstall common CRDS: %w", err) } return nil } - -// rolloutRestartContour will rollout restart the Contour pods in case any Contour configuration was changed. -func rolloutRestartContour(ctx context.Context, snap snap.Snap, attempts int) error { - client, err := snap.KubernetesClient("") - if err != nil { - return fmt.Errorf("failed to create kubernetes client: %w", err) - } - - if err := control.RetryFor(ctx, attempts, 0, func() error { - if err := client.RestartDeployment(ctx, "ck-ingress-contour-contour", "projectcontour"); err != nil { - return fmt.Errorf("failed to restart contour deployment: %w", err) - } - return nil - }); err != nil { - return fmt.Errorf("failed to restart contour deployment after %d attempts: %w", attempts, err) - } - - return nil -} diff --git a/src/k8s/pkg/k8sd/features/contour/ingress_test.go b/src/k8s/pkg/k8sd/features/contour/ingress/ingress_test.go similarity index 78% rename from src/k8s/pkg/k8sd/features/contour/ingress_test.go rename to src/k8s/pkg/k8sd/features/contour/ingress/ingress_test.go index e015f2671..03035db6c 100644 --- a/src/k8s/pkg/k8sd/features/contour/ingress_test.go +++ b/src/k8s/pkg/k8sd/features/contour/ingress/ingress_test.go @@ -1,4 +1,4 @@ -package contour_test +package ingress_test import ( "context" @@ -11,6 +11,7 @@ import ( helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/client/kubernetes" "github.com/canonical/k8s/pkg/k8sd/features/contour" + contour_ingress "github.com/canonical/k8s/pkg/k8sd/features/contour/ingress" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -40,13 +41,13 @@ func TestIngressDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(contour.IngressDeleteFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(contour_ingress.IngressDeleteFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -65,11 +66,11 @@ func TestIngressDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) g.Expect(status.Message).To(Equal(contour.DisabledMsg)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -94,13 +95,13 @@ func TestIngressEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(context.Background(), snapM, mc, ingress, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(contour.IngressDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(contour_ingress.IngressDeployFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -152,11 +153,11 @@ func TestIngressEnabled(t *testing.T) { defer cancel() mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(ctx, snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(ctx, snapM, mc, ingress, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) g.Expect(status.Message).To(Equal(contour.EnabledMsg)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(3)) validateIngressValues(g, helmM.ApplyCalledWith[1].Values, ingress) @@ -211,11 +212,11 @@ func TestIngressEnabled(t *testing.T) { defer cancel() mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(ctx, snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(ctx, snapM, mc, ingress, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) g.Expect(status.Message).To(Equal(contour.EnabledMsg)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(3)) validateIngressValues(g, helmM.ApplyCalledWith[1].Values, ingress) @@ -271,11 +272,11 @@ func TestIngressEnabled(t *testing.T) { defer cancel() mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(ctx, snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(ctx, snapM, mc, ingress, network, nil) g.Expect(err).NotTo(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) g.Expect(status.Message).To(Equal(contour.EnabledMsg)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(3)) validateIngressValues(g, helmM.ApplyCalledWith[1].Values, ingress) @@ -321,12 +322,12 @@ func TestIngressEnabled(t *testing.T) { defer cancel() mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(ctx, snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(ctx, snapM, mc, ingress, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(contour.IngressDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(contour_ingress.IngressDeployFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) }) @@ -378,13 +379,13 @@ func TestIngressEnabled(t *testing.T) { defer cancel() mc := snapM.HelmClient(loader.NewEmbedLoader(&contour.ChartFS)) - status, err := contour.ApplyIngress(ctx, snapM, mc, ingress, network, nil) + status, err := contour_ingress.ApplyIngress(ctx, snapM, mc, ingress, network, nil) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("failed to rollout restart contour to apply ingress")) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(contour.ContourIngressContourImageTag)) - g.Expect(status.Message).To(Equal(fmt.Sprintf(contour.IngressDeployFailedMsgTmpl, err))) + g.Expect(status.Version).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) + g.Expect(status.Message).To(Equal(fmt.Sprintf(contour_ingress.IngressDeployFailedMsgTmpl, err))) g.Expect(helmM.ApplyCalledWith).To(HaveLen(2)) }) } @@ -394,14 +395,14 @@ func validateIngressValues(g Gomega, values map[string]interface{}, ingress type g.Expect(ok).To(BeTrue()) contourImage, ok := contourValues["image"].(map[string]any) g.Expect(ok).To(BeTrue()) - g.Expect(contourImage["repository"]).To(Equal(contour.ContourIngressContourImageRepo)) - g.Expect(contourImage["tag"]).To(Equal(contour.ContourIngressContourImageTag)) + g.Expect(contourImage["repository"]).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).GetURI())) + g.Expect(contourImage["tag"]).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressContourImageName).Tag)) envoyValues, ok := values["envoy"].(map[string]any) g.Expect(ok).To(BeTrue()) envoyImage, ok := envoyValues["image"].(map[string]any) g.Expect(ok).To(BeTrue()) - g.Expect(envoyImage["repository"]).To(Equal(contour.ContourIngressEnvoyImageRepo)) - g.Expect(envoyImage["tag"]).To(Equal(contour.ContourIngressEnvoyImageTag)) + g.Expect(envoyImage["repository"]).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressEnvoyImageName).GetURI())) + g.Expect(envoyImage["tag"]).To(Equal(contour_ingress.FeatureIngress.GetImage(contour_ingress.ContourIngressEnvoyImageName).Tag)) if ingress.GetEnableProxyProtocol() { conturExtraValues, ok := values["contour"].(map[string]any) diff --git a/src/k8s/pkg/k8sd/features/contour/ingress/manifest.go b/src/k8s/pkg/k8sd/features/contour/ingress/manifest.go new file mode 100644 index 000000000..5b4737a6b --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/ingress/manifest.go @@ -0,0 +1,55 @@ +package ingress + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + ChartContourName = "contour" + ChartDefaultTLSName = "ck-ingress-tls" + ChartCommonContourCRDSName = "ck-contour-common" + + ContourIngressEnvoyImageName = "envoy-ingress" + ContourIngressContourImageName = "contour-ingress" +) + +var manifest = types.FeatureManifest{ + Name: "ingress", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + ChartContourName: { + Name: "contour", + Version: "17.0.4", + InstallName: "ck-ingress", + InstallNamespace: "projectcontour", + }, + ChartCommonContourCRDSName: { + Name: "ck-contour-common", + Version: "1.28.2", + InstallName: "ck-contour-common", + InstallNamespace: "projectcontour", + }, + ChartDefaultTLSName: { + Name: "ck-ingress-tls", + Version: "0.1.0", + InstallName: "ck-ingress-tls", + InstallNamespace: "projectcontour-root", + }, + }, + + Images: map[string]types.Image{ + ContourIngressEnvoyImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "bitnami/envoy", + Tag: "1.28.2-debian-12-r0", + }, + ContourIngressContourImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "bitnami/contour", + Tag: "1.28.2-debian-12-r4", + }, + }, +} + +var FeatureIngress types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/contour/ingress/register.go b/src/k8s/pkg/k8sd/features/contour/ingress/register.go new file mode 100644 index 000000000..21a846736 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/ingress/register.go @@ -0,0 +1,17 @@ +package ingress + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + contourIngressContourImage := FeatureIngress.GetImage(ContourIngressContourImageName) + contourIngressEnvoyImage := FeatureIngress.GetImage(ContourIngressEnvoyImageName) + + images.Register( + fmt.Sprintf("%s:%s", contourIngressEnvoyImage.GetURI(), contourIngressEnvoyImage.Tag), + fmt.Sprintf("%s:%s", contourIngressContourImage.GetURI(), contourIngressContourImage.Tag), + ) +} diff --git a/src/k8s/pkg/k8sd/features/contour/ingress/values.go b/src/k8s/pkg/k8sd/features/contour/ingress/values.go new file mode 100644 index 000000000..1982dc490 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/ingress/values.go @@ -0,0 +1,90 @@ +package ingress + +import ( + "fmt" + + "dario.cat/mergo" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +type Values map[string]any + +func (v Values) ApplyDefaultValues() error { + values := map[string]any{ + "envoy-service-namespace": "projectcontour", + "envoy-service-name": "envoy", + "contour": map[string]any{ + "manageCRDs": false, + "ingressClass": map[string]any{ + "name": "ck-ingress", + "create": true, + "default": true, + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyImageOverrides() error { + contourIngressContourImage := FeatureIngress.GetImage(ContourIngressContourImageName) + contourIngressEnvoyImage := FeatureIngress.GetImage(ContourIngressEnvoyImageName) + + values := map[string]any{ + "envoy": map[string]any{ + "image": map[string]any{ + "registry": "", + "repository": contourIngressEnvoyImage.GetURI(), + "tag": contourIngressEnvoyImage.Tag, + }, + }, + "contour": map[string]any{ + "image": map[string]any{ + "registry": "", + "repository": contourIngressContourImage.GetURI(), + "tag": contourIngressContourImage.Tag, + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +func (v Values) ApplyClusterConfiguration(ingress types.Ingress) error { + var values map[string]any + if ingress.GetEnableProxyProtocol() { + values = map[string]any{ + "contour": map[string]any{ + "extraArgs": []string{"--use-proxy-protocol"}, + }, + } + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge cluster configuration: %w", err) + } + + return nil +} + +type TLSValues map[string]any + +func (v TLSValues) ApplyClusterConfiguration(ingress types.Ingress) error { + values := map[string]any{ + "defaultTLSSecret": ingress.GetDefaultTLSSecret(), + } + + if err := mergo.Merge(&v, TLSValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge cluster configuration: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/contour/register.go b/src/k8s/pkg/k8sd/features/contour/register.go index 2ef83d53d..da61cef8a 100644 --- a/src/k8s/pkg/k8sd/features/contour/register.go +++ b/src/k8s/pkg/k8sd/features/contour/register.go @@ -1,19 +1,9 @@ package contour import ( - "fmt" - "github.com/canonical/k8s/pkg/k8sd/charts" - "github.com/canonical/k8s/pkg/k8sd/images" ) func init() { - images.Register( - fmt.Sprintf("%s:%s", ContourIngressEnvoyImageRepo, ContourIngressEnvoyImageTag), - fmt.Sprintf("%s:%s", ContourIngressContourImageRepo, ContourIngressContourImageTag), - fmt.Sprintf("%s:%s", ContourGatewayProvisionerContourImageRepo, ContourGatewayProvisionerContourImageTag), - fmt.Sprintf("%s:%s", ContourGatewayProvisionerEnvoyImageRepo, ContourGatewayProvisionerEnvoyImageTag), - ) - charts.Register(&ChartFS) } diff --git a/src/k8s/pkg/k8sd/features/contour/status.go b/src/k8s/pkg/k8sd/features/contour/status.go new file mode 100644 index 000000000..ea47ae81d --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/status.go @@ -0,0 +1,6 @@ +package contour + +const ( + EnabledMsg = "enabled" + DisabledMsg = "disabled" +) diff --git a/src/k8s/pkg/k8sd/features/contour/util.go b/src/k8s/pkg/k8sd/features/contour/util.go new file mode 100644 index 000000000..851c36fdb --- /dev/null +++ b/src/k8s/pkg/k8sd/features/contour/util.go @@ -0,0 +1,72 @@ +package contour + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils/control" +) + +// WaitForRequiredContourCommonCRDs waits for the required contour CRDs to be available +// by checking the API resources by group version. +func WaitForRequiredContourCommonCRDs(ctx context.Context, snap snap.Snap) error { + client, err := snap.KubernetesClient("") + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %w", err) + } + + return control.WaitUntilReady(ctx, func() (bool, error) { + resourcesV1Alpha, err := client.ListResourcesForGroupVersion("projectcontour.io/v1alpha1") + if err != nil { + // This error is expected if the group version is not yet deployed. + return false, nil + } + resourcesV1, err := client.ListResourcesForGroupVersion("projectcontour.io/v1") + if err != nil { + // This error is expected if the group version is not yet deployed. + return false, nil + } + + requiredCRDs := map[string]bool{ + "projectcontour.io/v1alpha1:contourconfigurations": true, + "projectcontour.io/v1alpha1:contourdeployments": true, + "projectcontour.io/v1alpha1:extensionservices": true, + "projectcontour.io/v1:tlscertificatedelegations": true, + "projectcontour.io/v1:httpproxies": true, + } + + requiredCount := len(requiredCRDs) + for _, resource := range resourcesV1Alpha.APIResources { + if _, exists := requiredCRDs[fmt.Sprintf("projectcontour.io/v1alpha1:%s", resource.Name)]; exists { + requiredCount-- + } + } + for _, resource := range resourcesV1.APIResources { + if _, exists := requiredCRDs[fmt.Sprintf("projectcontour.io/v1:%s", resource.Name)]; exists { + requiredCount-- + } + } + + return requiredCount == 0, nil + }) +} + +// RolloutRestartContour will rollout restart the Contour pods in case any Contour configuration was changed. +func RolloutRestartContour(ctx context.Context, snap snap.Snap, attempts int) error { + client, err := snap.KubernetesClient("") + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + if err := control.RetryFor(ctx, attempts, 0, func() error { + if err := client.RestartDeployment(ctx, "ck-ingress-contour-contour", "projectcontour"); err != nil { + return fmt.Errorf("failed to restart contour deployment: %w", err) + } + return nil + }); err != nil { + return fmt.Errorf("failed to restart contour deployment after %d attempts: %w", attempts, err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/coredns/chart.go b/src/k8s/pkg/k8sd/features/coredns/chart.go index 08528d193..215e6b874 100644 --- a/src/k8s/pkg/k8sd/features/coredns/chart.go +++ b/src/k8s/pkg/k8sd/features/coredns/chart.go @@ -2,25 +2,7 @@ package coredns import ( "embed" - - "github.com/canonical/k8s/pkg/client/helm" ) //go:embed all:charts var ChartFS embed.FS - -var ( - // chartCoreDNS represents manifests to deploy CoreDNS. - Chart = helm.InstallableChart{ - Name: "coredns", - Version: "1.36.2", - InstallName: "ck-dns", - InstallNamespace: "kube-system", - } - - // imageRepo is the image to use for CoreDNS. - imageRepo = "ghcr.io/canonical/coredns" - - // ImageTag is the tag to use for the CoreDNS image. - ImageTag = "1.11.4-ck1" -) diff --git a/src/k8s/pkg/k8sd/features/coredns/coredns.go b/src/k8s/pkg/k8sd/features/coredns/dns/coredns.go similarity index 60% rename from src/k8s/pkg/k8sd/features/coredns/coredns.go rename to src/k8s/pkg/k8sd/features/coredns/dns/coredns.go index d4af8a536..a57dead3e 100644 --- a/src/k8s/pkg/k8sd/features/coredns/coredns.go +++ b/src/k8s/pkg/k8sd/features/coredns/dns/coredns.go @@ -1,9 +1,8 @@ -package coredns +package dns import ( "context" "fmt" - "strings" "github.com/canonical/k8s/pkg/client/helm" "github.com/canonical/k8s/pkg/k8sd/types" @@ -26,69 +25,49 @@ const ( // ApplyDNS returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. func ApplyDNS(ctx context.Context, snap snap.Snap, m helm.Client, dns types.DNS, kubelet types.Kubelet, _ types.Annotations) (types.FeatureStatus, string, error) { + coreDNSImage := FeatureDNS.GetImage(CoreDNSImageName) + if !dns.GetEnabled() { - if _, err := m.Apply(ctx, Chart, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureDNS.GetChart(CoreDNSChartName), helm.StateDeleted, nil); err != nil { err = fmt.Errorf("failed to uninstall coredns: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ImageTag, + Version: coreDNSImage.Tag, Message: fmt.Sprintf(deleteFailedMsgTmpl, err), }, "", err } return types.FeatureStatus{ Enabled: false, - Version: ImageTag, + Version: coreDNSImage.Tag, Message: disabledMsg, }, "", nil } - values := map[string]any{ - "image": map[string]any{ - "repository": imageRepo, - "tag": ImageTag, - }, - "service": map[string]any{ - "name": "coredns", - "clusterIP": kubelet.GetClusterDNS(), - }, - "serviceAccount": map[string]any{ - "create": true, - "name": "coredns", - }, - "deployment": map[string]any{ - "name": "coredns", - }, - "servers": []map[string]any{ - { - "zones": []map[string]any{ - {"zone": "."}, - }, - "port": 53, - "plugins": []map[string]any{ - {"name": "errors"}, - {"name": "health", "configBlock": "lameduck 5s"}, - {"name": "ready"}, - { - "name": "kubernetes", - "parameters": fmt.Sprintf("%s in-addr.arpa ip6.arpa", kubelet.GetClusterDomain()), - "configBlock": "pods insecure\nfallthrough in-addr.arpa ip6.arpa\nttl 30", - }, - {"name": "prometheus", "parameters": "0.0.0.0:9153"}, - {"name": "forward", "parameters": fmt.Sprintf(". %s", strings.Join(dns.GetUpstreamNameservers(), " "))}, - {"name": "cache", "parameters": "30"}, - {"name": "loop"}, - {"name": "reload"}, - {"name": "loadbalance"}, - }, - }, - }, + var values Values = map[string]any{} + + if err := values.ApplyImageOverrides(); err != nil { + err = fmt.Errorf("failed to apply image overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: coreDNSImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, "", err + } + + if err := values.ApplyClusterConfiguration(dns, kubelet); err != nil { + err = fmt.Errorf("failed to apply cluster configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: coreDNSImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, "", err } - if _, err := m.Apply(ctx, Chart, helm.StatePresent, values); err != nil { + if _, err := m.Apply(ctx, FeatureDNS.GetChart(CoreDNSChartName), helm.StatePresent, values); err != nil { err = fmt.Errorf("failed to apply coredns: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ImageTag, + Version: coreDNSImage.Tag, Message: fmt.Sprintf(deployFailedMsgTmpl, err), }, "", err } @@ -98,7 +77,7 @@ func ApplyDNS(ctx context.Context, snap snap.Snap, m helm.Client, dns types.DNS, err = fmt.Errorf("failed to create kubernetes client: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ImageTag, + Version: coreDNSImage.Tag, Message: fmt.Sprintf(deployFailedMsgTmpl, err), }, "", err } @@ -107,14 +86,14 @@ func ApplyDNS(ctx context.Context, snap snap.Snap, m helm.Client, dns types.DNS, err = fmt.Errorf("failed to retrieve the coredns service: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ImageTag, + Version: coreDNSImage.Tag, Message: fmt.Sprintf(deployFailedMsgTmpl, err), }, "", err } return types.FeatureStatus{ Enabled: true, - Version: ImageTag, + Version: coreDNSImage.Tag, Message: fmt.Sprintf(enabledMsgTmpl, dnsIP), }, dnsIP, err } diff --git a/src/k8s/pkg/k8sd/features/coredns/coredns_test.go b/src/k8s/pkg/k8sd/features/coredns/dns/coredns_test.go similarity index 77% rename from src/k8s/pkg/k8sd/features/coredns/coredns_test.go rename to src/k8s/pkg/k8sd/features/coredns/dns/coredns_test.go index 1d52d16c7..c0767adf1 100644 --- a/src/k8s/pkg/k8sd/features/coredns/coredns_test.go +++ b/src/k8s/pkg/k8sd/features/coredns/dns/coredns_test.go @@ -1,4 +1,4 @@ -package coredns_test +package dns_test import ( "context" @@ -11,6 +11,7 @@ import ( helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/client/kubernetes" "github.com/canonical/k8s/pkg/k8sd/features/coredns" + coredns_dns "github.com/canonical/k8s/pkg/k8sd/features/coredns/dns" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -40,17 +41,17 @@ func TestDisabled(t *testing.T) { mc := snapM.HelmClient(loader.NewEmbedLoader(&coredns.ChartFS)) - status, str, err := coredns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) + status, str, err := coredns_dns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) g.Expect(err).To(MatchError(ContainSubstring(applyErr.Error()))) g.Expect(str).To(BeEmpty()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) g.Expect(status.Message).To(ContainSubstring("failed to uninstall coredns")) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(coredns.ImageTag)) + g.Expect(status.Version).To(Equal(coredns_dns.FeatureDNS.GetImage(coredns_dns.CoreDNSImageName).Tag)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(coredns.Chart)) + g.Expect(callArgs.Chart).To(Equal(coredns_dns.FeatureDNS.GetChart(coredns_dns.CoreDNSChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -69,17 +70,17 @@ func TestDisabled(t *testing.T) { kubelet := types.Kubelet{} mc := snapM.HelmClient(loader.NewEmbedLoader(&coredns.ChartFS)) - status, str, err := coredns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) + status, str, err := coredns_dns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) g.Expect(err).To(Not(HaveOccurred())) g.Expect(str).To(BeEmpty()) g.Expect(status.Message).To(Equal("disabled")) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(coredns.ImageTag)) + g.Expect(status.Version).To(Equal(coredns_dns.FeatureDNS.GetImage(coredns_dns.CoreDNSImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(coredns.Chart)) + g.Expect(callArgs.Chart).To(Equal(coredns_dns.FeatureDNS.GetChart(coredns_dns.CoreDNSChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -105,17 +106,17 @@ func TestEnabled(t *testing.T) { mc := snapM.HelmClient(loader.NewEmbedLoader(&coredns.ChartFS)) - status, str, err := coredns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) + status, str, err := coredns_dns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) g.Expect(err).To(MatchError(ContainSubstring(applyErr.Error()))) g.Expect(str).To(BeEmpty()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) g.Expect(status.Message).To(ContainSubstring("failed to apply coredns")) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(coredns.ImageTag)) + g.Expect(status.Version).To(Equal(coredns_dns.FeatureDNS.GetImage(coredns_dns.CoreDNSImageName).Tag)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(coredns.Chart)) + g.Expect(callArgs.Chart).To(Equal(coredns_dns.FeatureDNS.GetChart(coredns_dns.CoreDNSChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateValues(g, callArgs.Values, dns, kubelet) }) @@ -137,17 +138,17 @@ func TestEnabled(t *testing.T) { mc := snapM.HelmClient(loader.NewEmbedLoader(&coredns.ChartFS)) - status, str, err := coredns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) + status, str, err := coredns_dns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) g.Expect(err).To(MatchError(ContainSubstring("services \"coredns\" not found"))) g.Expect(str).To(BeEmpty()) g.Expect(status.Message).To(ContainSubstring("failed to retrieve the coredns service")) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(coredns.ImageTag)) + g.Expect(status.Version).To(Equal(coredns_dns.FeatureDNS.GetImage(coredns_dns.CoreDNSImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(coredns.Chart)) + g.Expect(callArgs.Chart).To(Equal(coredns_dns.FeatureDNS.GetChart(coredns_dns.CoreDNSChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateValues(g, callArgs.Values, dns, kubelet) }) @@ -178,17 +179,17 @@ func TestEnabled(t *testing.T) { kubelet := types.Kubelet{} mc := snapM.HelmClient(loader.NewEmbedLoader(&coredns.ChartFS)) - status, str, err := coredns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) + status, str, err := coredns_dns.ApplyDNS(context.Background(), snapM, mc, dns, kubelet, nil) g.Expect(err).To(Not(HaveOccurred())) g.Expect(str).To(Equal(clusterIp)) g.Expect(status.Message).To(ContainSubstring("enabled at " + clusterIp)) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(coredns.ImageTag)) + g.Expect(status.Version).To(Equal(coredns_dns.FeatureDNS.GetImage(coredns_dns.CoreDNSImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(coredns.Chart)) + g.Expect(callArgs.Chart).To(Equal(coredns_dns.FeatureDNS.GetChart(coredns_dns.CoreDNSChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateValues(g, callArgs.Values, dns, kubelet) }) diff --git a/src/k8s/pkg/k8sd/features/coredns/dns/manifest.go b/src/k8s/pkg/k8sd/features/coredns/dns/manifest.go new file mode 100644 index 000000000..39efb4376 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/coredns/dns/manifest.go @@ -0,0 +1,34 @@ +package dns + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + CoreDNSChartName = "coredns" + CoreDNSImageName = "coredns" +) + +var manifest = types.FeatureManifest{ + Name: "dns", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + CoreDNSChartName: { + Name: "coredns", + Version: "1.36.2", + InstallName: "ck-dns", + InstallNamespace: "kube-system", + }, + }, + + Images: map[string]types.Image{ + CoreDNSImageName: { + Registry: "ghcr.io/canonical", + Repository: "coredns", + Tag: "1.11.4-ck1", + }, + }, +} + +var FeatureDNS types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/coredns/dns/register.go b/src/k8s/pkg/k8sd/features/coredns/dns/register.go new file mode 100644 index 000000000..ecf709715 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/coredns/dns/register.go @@ -0,0 +1,15 @@ +package dns + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + coreDNSImage := FeatureDNS.GetImage(CoreDNSImageName) + + images.Register( + fmt.Sprintf("%s:%s", coreDNSImage.GetURI(), coreDNSImage.Tag), + ) +} diff --git a/src/k8s/pkg/k8sd/features/coredns/dns/values.go b/src/k8s/pkg/k8sd/features/coredns/dns/values.go new file mode 100644 index 000000000..e77bc2a6c --- /dev/null +++ b/src/k8s/pkg/k8sd/features/coredns/dns/values.go @@ -0,0 +1,74 @@ +package dns + +import ( + "fmt" + "strings" + + "dario.cat/mergo" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +type Values map[string]any + +func (v Values) ApplyImageOverrides() error { + coreDNSImage := FeatureDNS.GetImage(CoreDNSImageName) + + values := map[string]any{ + "image": map[string]any{ + "repository": coreDNSImage.GetURI(), + "tag": coreDNSImage.Tag, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +func (v Values) ApplyClusterConfiguration(dns types.DNS, kubelet types.Kubelet) error { + values := map[string]any{ + "service": map[string]any{ + "name": "coredns", + "clusterIP": kubelet.GetClusterDNS(), + }, + "serviceAccount": map[string]any{ + "create": true, + "name": "coredns", + }, + "deployment": map[string]any{ + "name": "coredns", + }, + "servers": []map[string]any{ + { + "zones": []map[string]any{ + {"zone": "."}, + }, + "port": 53, + "plugins": []map[string]any{ + {"name": "errors"}, + {"name": "health", "configBlock": "lameduck 5s"}, + {"name": "ready"}, + { + "name": "kubernetes", + "parameters": fmt.Sprintf("%s in-addr.arpa ip6.arpa", kubelet.GetClusterDomain()), + "configBlock": "pods insecure\nfallthrough in-addr.arpa ip6.arpa\nttl 30", + }, + {"name": "prometheus", "parameters": "0.0.0.0:9153"}, + {"name": "forward", "parameters": fmt.Sprintf(". %s", strings.Join(dns.GetUpstreamNameservers(), " "))}, + {"name": "cache", "parameters": "30"}, + {"name": "loop"}, + {"name": "reload"}, + {"name": "loadbalance"}, + }, + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge cluster configuration: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/coredns/register.go b/src/k8s/pkg/k8sd/features/coredns/register.go index a52ba19ed..a898fa4b1 100644 --- a/src/k8s/pkg/k8sd/features/coredns/register.go +++ b/src/k8s/pkg/k8sd/features/coredns/register.go @@ -1,16 +1,9 @@ package coredns import ( - "fmt" - "github.com/canonical/k8s/pkg/k8sd/charts" - "github.com/canonical/k8s/pkg/k8sd/images" ) func init() { - images.Register( - fmt.Sprintf("%s:%s", imageRepo, ImageTag), - ) - charts.Register(&ChartFS) } diff --git a/src/k8s/pkg/k8sd/features/implementation_default.go b/src/k8s/pkg/k8sd/features/implementation_default.go index 3cad3e3b6..6b1f6393e 100644 --- a/src/k8s/pkg/k8sd/features/implementation_default.go +++ b/src/k8s/pkg/k8sd/features/implementation_default.go @@ -2,9 +2,13 @@ package features import ( "github.com/canonical/k8s/pkg/k8sd/features/cilium" + cilium_gateway "github.com/canonical/k8s/pkg/k8sd/features/cilium/gateway" + cilium_ingress "github.com/canonical/k8s/pkg/k8sd/features/cilium/ingress" + cilium_network "github.com/canonical/k8s/pkg/k8sd/features/cilium/network" "github.com/canonical/k8s/pkg/k8sd/features/coredns" - "github.com/canonical/k8s/pkg/k8sd/features/localpv" - "github.com/canonical/k8s/pkg/k8sd/features/metallb" + coredns_dns "github.com/canonical/k8s/pkg/k8sd/features/coredns/dns" + localpv_local_storage "github.com/canonical/k8s/pkg/k8sd/features/localpv/local-storage" + metallb_loadbalancer "github.com/canonical/k8s/pkg/k8sd/features/metallb/loadbalancer" metrics_server "github.com/canonical/k8s/pkg/k8sd/features/metrics-server" ) @@ -15,13 +19,13 @@ import ( // MetricsServer is used for metrics-server. // LocalPV Rawfile CSI is used for local-storage. var Implementation Interface = &implementation{ - applyDNS: coredns.ApplyDNS, - applyNetwork: cilium.ApplyNetwork, - applyLoadBalancer: metallb.ApplyLoadBalancer, - applyIngress: cilium.ApplyIngress, - applyGateway: cilium.ApplyGateway, + applyDNS: coredns_dns.ApplyDNS, + applyNetwork: cilium_network.ApplyNetwork, + applyLoadBalancer: metallb_loadbalancer.ApplyLoadBalancer, + applyIngress: cilium_ingress.ApplyIngress, + applyGateway: cilium_gateway.ApplyGateway, applyMetricsServer: metrics_server.ApplyMetricsServer, - applyLocalStorage: localpv.ApplyLocalStorage, + applyLocalStorage: localpv_local_storage.ApplyLocalStorage, } // StatusChecks implements the Canonical Kubernetes built-in feature status checks. diff --git a/src/k8s/pkg/k8sd/features/implementation_moonray.go b/src/k8s/pkg/k8sd/features/implementation_moonray.go index cbaacad91..f5c4d3dc4 100644 --- a/src/k8s/pkg/k8sd/features/implementation_moonray.go +++ b/src/k8s/pkg/k8sd/features/implementation_moonray.go @@ -4,23 +4,26 @@ package features import ( "github.com/canonical/k8s/pkg/k8sd/features/calico" - "github.com/canonical/k8s/pkg/k8sd/features/contour" + calico_network "github.com/canonical/k8s/pkg/k8sd/features/calico/network" + contour_gateway "github.com/canonical/k8s/pkg/k8sd/features/contour/gateway" + contour_ingress "github.com/canonical/k8s/pkg/k8sd/features/contour/ingress" "github.com/canonical/k8s/pkg/k8sd/features/coredns" - "github.com/canonical/k8s/pkg/k8sd/features/localpv" - "github.com/canonical/k8s/pkg/k8sd/features/metallb" + coredns_dns "github.com/canonical/k8s/pkg/k8sd/features/coredns/dns" + localpv_local_storage "github.com/canonical/k8s/pkg/k8sd/features/localpv/local-storage" + metallb_loadbalancer "github.com/canonical/k8s/pkg/k8sd/features/metallb/loadbalancer" metrics_server "github.com/canonical/k8s/pkg/k8sd/features/metrics-server" ) // Implementation contains the moonray features for Canonical Kubernetes. // TODO: Replace default by moonray. var Implementation Interface = &implementation{ - applyDNS: coredns.ApplyDNS, - applyNetwork: calico.ApplyNetwork, - applyLoadBalancer: metallb.ApplyLoadBalancer, - applyIngress: contour.ApplyIngress, - applyGateway: contour.ApplyGateway, + applyDNS: coredns_dns.ApplyDNS, + applyNetwork: calico_network.ApplyNetwork, + applyLoadBalancer: metallb_loadbalancer.ApplyLoadBalancer, + applyIngress: contour_ingress.ApplyIngress, + applyGateway: contour_gateway.ApplyGateway, applyMetricsServer: metrics_server.ApplyMetricsServer, - applyLocalStorage: localpv.ApplyLocalStorage, + applyLocalStorage: localpv_local_storage.ApplyLocalStorage, } // StatusChecks implements the Canonical Kubernetes moonray feature status checks. diff --git a/src/k8s/pkg/k8sd/features/localpv/chart.go b/src/k8s/pkg/k8sd/features/localpv/chart.go index 34b846629..455a0e94f 100644 --- a/src/k8s/pkg/k8sd/features/localpv/chart.go +++ b/src/k8s/pkg/k8sd/features/localpv/chart.go @@ -2,33 +2,7 @@ package localpv import ( "embed" - - "github.com/canonical/k8s/pkg/client/helm" ) //go:embed all:charts var ChartFS embed.FS - -var ( - // Chart represents manifests to deploy Rawfile LocalPV CSI. - Chart = helm.InstallableChart{ - Name: "rawfile-csi", - Version: "0.9.0", - InstallName: "ck-storage", - InstallNamespace: "kube-system", - } - - // imageRepo is the repository to use for Rawfile LocalPV CSI. - imageRepo = "ghcr.io/canonical/rawfile-localpv" - // ImageTag is the image tag to use for Rawfile LocalPV CSI. - ImageTag = "0.8.1" - - // csiNodeDriverImage is the image to use for the CSI node driver. - csiNodeDriverImage = "ghcr.io/canonical/k8s-snap/sig-storage/csi-node-driver-registrar:v2.10.1" - // csiProvisionerImage is the image to use for the CSI provisioner. - csiProvisionerImage = "ghcr.io/canonical/k8s-snap/sig-storage/csi-provisioner:v5.0.2" - // csiResizerImage is the image to use for the CSI resizer. - csiResizerImage = "ghcr.io/canonical/k8s-snap/sig-storage/csi-resizer:v1.11.2" - // csiSnapshotterImage is the image to use for the CSI snapshotter. - csiSnapshotterImage = "ghcr.io/canonical/k8s-snap/sig-storage/csi-snapshotter:v8.0.2" -) diff --git a/src/k8s/pkg/k8sd/features/localpv/localpv.go b/src/k8s/pkg/k8sd/features/localpv/local-storage/localpv.go similarity index 52% rename from src/k8s/pkg/k8sd/features/localpv/localpv.go rename to src/k8s/pkg/k8sd/features/localpv/local-storage/localpv.go index 25dfe8103..4d7603766 100644 --- a/src/k8s/pkg/k8sd/features/localpv/localpv.go +++ b/src/k8s/pkg/k8sd/features/localpv/local-storage/localpv.go @@ -1,17 +1,16 @@ -package localpv +package local_storage import ( "context" "fmt" "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/localpv" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" ) const ( - enabledMsg = "enabled at %s" - disabledMsg = "disabled" deployFailedMsgTmpl = "Failed to deploy Local Storage, the error was: %v" deleteFailedMsgTmpl = "Failed to delete Local Storage, the error was: %v" ) @@ -23,52 +22,50 @@ const ( // ApplyLocalStorage returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. func ApplyLocalStorage(ctx context.Context, snap snap.Snap, m helm.Client, cfg types.LocalStorage, _ types.Annotations) (types.FeatureStatus, error) { - values := map[string]any{ - "storageClass": map[string]any{ - "enabled": true, - "isDefault": cfg.GetDefault(), - "reclaimPolicy": cfg.GetReclaimPolicy(), - }, - "serviceMonitor": map[string]any{ - "enabled": false, - }, - "controller": map[string]any{ - "csiDriverArgs": []string{"--args", "rawfile", "csi-driver", "--disable-metrics"}, - "image": map[string]any{ - "repository": imageRepo, - "tag": ImageTag, - }, - }, - "node": map[string]any{ - "image": map[string]any{ - "repository": imageRepo, - "tag": ImageTag, - }, - "storage": map[string]any{ - "path": cfg.GetLocalPath(), - }, - }, - "images": map[string]any{ - "csiNodeDriverRegistrar": csiNodeDriverImage, - "csiProvisioner": csiProvisionerImage, - "csiResizer": csiResizerImage, - "csiSnapshotter": csiSnapshotterImage, - }, + rawFileImage := FeatureLocalStorage.GetImage(RawFileImageName) + + var values Values = map[string]any{} + + if err := values.ApplyDefaultValues(); err != nil { + err = fmt.Errorf("failed to apply default values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: rawFileImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyImageOverrides(); err != nil { + err = fmt.Errorf("failed to apply image overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: rawFileImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyClusterConfiguration(cfg); err != nil { + err = fmt.Errorf("failed to apply cluster configuration: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: rawFileImage.Tag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err } - if _, err := m.Apply(ctx, Chart, helm.StatePresentOrDeleted(cfg.GetEnabled()), values); err != nil { + if _, err := m.Apply(ctx, FeatureLocalStorage.GetChart(RawFileChartName), helm.StatePresentOrDeleted(cfg.GetEnabled()), values); err != nil { if cfg.GetEnabled() { err = fmt.Errorf("failed to install rawfile-csi helm package: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ImageTag, + Version: rawFileImage.Tag, Message: fmt.Sprintf(deployFailedMsgTmpl, err), }, err } else { err = fmt.Errorf("failed to delete rawfile-csi helm package: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ImageTag, + Version: rawFileImage.Tag, Message: fmt.Sprintf(deleteFailedMsgTmpl, err), }, err } @@ -77,14 +74,14 @@ func ApplyLocalStorage(ctx context.Context, snap snap.Snap, m helm.Client, cfg t if cfg.GetEnabled() { return types.FeatureStatus{ Enabled: true, - Version: ImageTag, - Message: fmt.Sprintf(enabledMsg, cfg.GetLocalPath()), + Version: rawFileImage.Tag, + Message: fmt.Sprintf(localpv.EnabledMsg, cfg.GetLocalPath()), }, nil } else { return types.FeatureStatus{ Enabled: false, - Version: ImageTag, - Message: disabledMsg, + Version: rawFileImage.Tag, + Message: localpv.DisabledMsg, }, nil } } diff --git a/src/k8s/pkg/k8sd/features/localpv/localpv_test.go b/src/k8s/pkg/k8sd/features/localpv/local-storage/localpv_test.go similarity index 71% rename from src/k8s/pkg/k8sd/features/localpv/localpv_test.go rename to src/k8s/pkg/k8sd/features/localpv/local-storage/localpv_test.go index cdad1a9d4..516152fac 100644 --- a/src/k8s/pkg/k8sd/features/localpv/localpv_test.go +++ b/src/k8s/pkg/k8sd/features/localpv/local-storage/localpv_test.go @@ -1,4 +1,4 @@ -package localpv_test +package local_storage_test import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/canonical/k8s/pkg/client/helm/loader" helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/k8sd/features/localpv" + localpv_local_storage "github.com/canonical/k8s/pkg/k8sd/features/localpv/local-storage" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -36,16 +37,16 @@ func TestDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&localpv.ChartFS)) - status, err := localpv.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) + status, err := localpv_local_storage.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) - g.Expect(status.Version).To(Equal(localpv.ImageTag)) + g.Expect(status.Version).To(Equal(localpv_local_storage.FeatureLocalStorage.GetImage(localpv_local_storage.RawFileImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(localpv.Chart)) + g.Expect(callArgs.Chart).To(Equal(localpv_local_storage.FeatureLocalStorage.GetChart(localpv_local_storage.RawFileChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) validateValues(g, callArgs.Values, cfg) @@ -67,15 +68,15 @@ func TestDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&localpv.ChartFS)) - status, err := localpv.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) + status, err := localpv_local_storage.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) - g.Expect(status.Version).To(Equal(localpv.ImageTag)) + g.Expect(status.Version).To(Equal(localpv_local_storage.FeatureLocalStorage.GetImage(localpv_local_storage.RawFileImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(localpv.Chart)) + g.Expect(callArgs.Chart).To(Equal(localpv_local_storage.FeatureLocalStorage.GetChart(localpv_local_storage.RawFileChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) validateValues(g, callArgs.Values, cfg) @@ -103,16 +104,16 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&localpv.ChartFS)) - status, err := localpv.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) + status, err := localpv_local_storage.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) - g.Expect(status.Version).To(Equal(localpv.ImageTag)) + g.Expect(status.Version).To(Equal(localpv_local_storage.FeatureLocalStorage.GetImage(localpv_local_storage.RawFileImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(localpv.Chart)) + g.Expect(callArgs.Chart).To(Equal(localpv_local_storage.FeatureLocalStorage.GetChart(localpv_local_storage.RawFileChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateValues(g, callArgs.Values, cfg) @@ -134,15 +135,15 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&localpv.ChartFS)) - status, err := localpv.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) + status, err := localpv_local_storage.ApplyLocalStorage(context.Background(), snapM, mc, cfg, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(localpv.ImageTag)) + g.Expect(status.Version).To(Equal(localpv_local_storage.FeatureLocalStorage.GetImage(localpv_local_storage.RawFileImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(localpv.Chart)) + g.Expect(callArgs.Chart).To(Equal(localpv_local_storage.FeatureLocalStorage.GetChart(localpv_local_storage.RawFileChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) validateValues(g, callArgs.Values, cfg) diff --git a/src/k8s/pkg/k8sd/features/localpv/local-storage/manifest.go b/src/k8s/pkg/k8sd/features/localpv/local-storage/manifest.go new file mode 100644 index 000000000..6c1ddb8e5 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/localpv/local-storage/manifest.go @@ -0,0 +1,59 @@ +package local_storage + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + RawFileChartName = "rawfile-csi" + + RawFileImageName = "rawfile-localpv" + CSINodeDriverImageName = "csi-node-driver-registrar" + CSIProvisionerImageName = "csi-provisioner" + CSIResizerImageName = "csi-resizer" + CSISnapshotterImageName = "csi-snapshotter" +) + +var manifest = types.FeatureManifest{ + Name: "local-storage", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + RawFileChartName: { + Name: "rawfile-csi", + Version: "0.9.0", + InstallName: "ck-storage", + InstallNamespace: "kube-system", + }, + }, + + Images: map[string]types.Image{ + RawFileImageName: { + Registry: "ghcr.io/canonical", + Repository: "rawfile-localpv", + Tag: "0.8.1", + }, + CSINodeDriverImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "sig-storage/csi-node-driver-registrar", + Tag: "v2.10.1", + }, + CSIProvisionerImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "sig-storage/csi-provisioner", + Tag: "v5.0.1", + }, + CSIResizerImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "sig-storage/csi-resizer", + Tag: "v1.11.1", + }, + CSISnapshotterImageName: { + Registry: "ghcr.io/canonical/k8s-snap", + Repository: "sig-storage/csi-snapshotter", + Tag: "v8.0.1", + }, + }, +} + +var FeatureLocalStorage types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/localpv/local-storage/register.go b/src/k8s/pkg/k8sd/features/localpv/local-storage/register.go new file mode 100644 index 000000000..965980d80 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/localpv/local-storage/register.go @@ -0,0 +1,25 @@ +package local_storage + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + rawFileImage := FeatureLocalStorage.GetImage(RawFileImageName) + csiNodeDriverImage := FeatureLocalStorage.GetImage(CSINodeDriverImageName) + csiProvisionerImage := FeatureLocalStorage.GetImage(CSIProvisionerImageName) + csiResizerImage := FeatureLocalStorage.GetImage(CSIResizerImageName) + csiSnapshotterImage := FeatureLocalStorage.GetImage(CSISnapshotterImageName) + + images.Register( + // Rawfile LocalPV CSI driver images + fmt.Sprintf("%s:%s", rawFileImage.GetURI(), rawFileImage.Tag), + // CSI images + fmt.Sprintf("%s:%s", csiNodeDriverImage.GetURI(), csiNodeDriverImage.Tag), + fmt.Sprintf("%s:%s", csiProvisionerImage.GetURI(), csiProvisionerImage.Tag), + fmt.Sprintf("%s:%s", csiResizerImage.GetURI(), csiResizerImage.Tag), + fmt.Sprintf("%s:%s", csiSnapshotterImage.GetURI(), csiSnapshotterImage.Tag), + ) +} diff --git a/src/k8s/pkg/k8sd/features/localpv/local-storage/values.go b/src/k8s/pkg/k8sd/features/localpv/local-storage/values.go new file mode 100644 index 000000000..2bc46dd60 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/localpv/local-storage/values.go @@ -0,0 +1,85 @@ +package local_storage + +import ( + "fmt" + + "dario.cat/mergo" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +type Values map[string]any + +func (v Values) ApplyDefaultValues() error { + values := map[string]any{ + "storageClass": map[string]any{ + "enabled": true, + }, + "serviceMonitor": map[string]any{ + "enabled": false, + }, + "controller": map[string]any{ + "csiDriverArgs": []string{"--args", "rawfile", "csi-driver", "--disable-metrics"}, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyImageOverrides() error { + rawFileImage := FeatureLocalStorage.GetImage(RawFileImageName) + csiNodeDriverImage := FeatureLocalStorage.GetImage(CSINodeDriverImageName) + csiProvisionerImage := FeatureLocalStorage.GetImage(CSIProvisionerImageName) + csiResizerImage := FeatureLocalStorage.GetImage(CSIResizerImageName) + csiSnapshotterImage := FeatureLocalStorage.GetImage(CSISnapshotterImageName) + + values := map[string]any{ + "controller": map[string]any{ + "image": map[string]any{ + "repository": rawFileImage.GetURI(), + "tag": rawFileImage.Tag, + }, + }, + "node": map[string]any{ + "image": map[string]any{ + "repository": rawFileImage.GetURI(), + "tag": rawFileImage.Tag, + }, + }, + "images": map[string]any{ + "csiNodeDriverRegistrar": fmt.Sprintf("%s:%s", csiNodeDriverImage.GetURI(), csiNodeDriverImage.Tag), + "csiProvisioner": fmt.Sprintf("%s:%s", csiProvisionerImage.GetURI(), csiProvisionerImage.Tag), + "csiResizer": fmt.Sprintf("%s:%s", csiResizerImage.GetURI(), csiResizerImage.Tag), + "csiSnapshotter": fmt.Sprintf("%s:%s", csiSnapshotterImage.GetURI(), csiSnapshotterImage.Tag), + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +func (v Values) ApplyClusterConfiguration(cfg types.LocalStorage) error { + values := map[string]any{ + "storageClass": map[string]any{ + "isDefault": cfg.GetDefault(), + "reclaimPolicy": cfg.GetReclaimPolicy(), + }, + "node": map[string]any{ + "storage": map[string]any{ + "path": cfg.GetLocalPath(), + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge cluster configuration values: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/localpv/register.go b/src/k8s/pkg/k8sd/features/localpv/register.go index 06475664e..93c945d35 100644 --- a/src/k8s/pkg/k8sd/features/localpv/register.go +++ b/src/k8s/pkg/k8sd/features/localpv/register.go @@ -1,22 +1,9 @@ package localpv import ( - "fmt" - "github.com/canonical/k8s/pkg/k8sd/charts" - "github.com/canonical/k8s/pkg/k8sd/images" ) func init() { - images.Register( - // Rawfile LocalPV CSI driver images - fmt.Sprintf("%s:%s", imageRepo, ImageTag), - // CSI images - csiNodeDriverImage, - csiProvisionerImage, - csiResizerImage, - csiSnapshotterImage, - ) - charts.Register(&ChartFS) } diff --git a/src/k8s/pkg/k8sd/features/localpv/status.go b/src/k8s/pkg/k8sd/features/localpv/status.go new file mode 100644 index 000000000..1f257482e --- /dev/null +++ b/src/k8s/pkg/k8sd/features/localpv/status.go @@ -0,0 +1,6 @@ +package localpv + +const ( + EnabledMsg = "enabled at %s" + DisabledMsg = "disabled" +) diff --git a/src/k8s/pkg/k8sd/features/metallb/chart.go b/src/k8s/pkg/k8sd/features/metallb/chart.go index c92ed4252..0bbe081ce 100644 --- a/src/k8s/pkg/k8sd/features/metallb/chart.go +++ b/src/k8s/pkg/k8sd/features/metallb/chart.go @@ -2,45 +2,7 @@ package metallb import ( "embed" - - "github.com/canonical/k8s/pkg/client/helm" ) //go:embed all:charts var ChartFS embed.FS - -var ( - // ChartMetalLB represents manifests to deploy MetalLB speaker and controller. - ChartMetalLB = helm.InstallableChart{ - Name: "metallb", - Version: "0.14.9", - InstallName: "metallb", - InstallNamespace: "metallb-system", - } - - // ChartMetalLBLoadBalancer represents manifests to deploy MetalLB L2 or BGP resources. - ChartMetalLBLoadBalancer = helm.InstallableChart{ - Name: "ck-loadbalancer", - Version: "0.1.1", - InstallName: "metallb-loadbalancer", - InstallNamespace: "metallb-system", - } - - // controllerImageRepo is the image to use for metallb-controller. - controllerImageRepo = "ghcr.io/canonical/metallb-controller" - - // ControllerImageTag is the tag to use for metallb-controller. - ControllerImageTag = "v0.14.9-ck0" - - // speakerImageRepo is the image to use for metallb-speaker. - speakerImageRepo = "ghcr.io/canonical/metallb-speaker" - - // speakerImageTag is the tag to use for metallb-speaker. - speakerImageTag = "v0.14.9-ck0" - - // frrImageRepo is the image to use for frrouting. - frrImageRepo = "ghcr.io/canonical/frr" - - // frrImageTag is the tag to use for frrouting. - frrImageTag = "9.1.3" -) diff --git a/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/loadbalancer.go similarity index 62% rename from src/k8s/pkg/k8sd/features/metallb/loadbalancer.go rename to src/k8s/pkg/k8sd/features/metallb/loadbalancer/loadbalancer.go index 8a4108353..556b12f1f 100644 --- a/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go +++ b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/loadbalancer.go @@ -1,18 +1,17 @@ -package metallb +package loadbalancer import ( "context" "fmt" "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/features/metallb" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" "github.com/canonical/k8s/pkg/utils/control" ) const ( - enabledMsgTmpl = "enabled, %s mode" - DisabledMsg = "disabled" deleteFailedMsgTmpl = "Failed to delete MetalLB, the error was: %v" deployFailedMsgTmpl = "Failed to deploy MetalLB, the error was: %v" ) @@ -22,19 +21,21 @@ const ( // ApplyLoadBalancer returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadbalancer types.LoadBalancer, network types.Network, _ types.Annotations) (types.FeatureStatus, error) { + metalLBControllerImage := FeatureLoadBalancer.GetImage(MetalLBControllerImageName) + if !loadbalancer.GetEnabled() { if err := disableLoadBalancer(ctx, snap, m, network); err != nil { err = fmt.Errorf("failed to disable LoadBalancer: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ControllerImageTag, + Version: metalLBControllerImage.Tag, Message: fmt.Sprintf(deleteFailedMsgTmpl, err), }, err } return types.FeatureStatus{ Enabled: false, - Version: ControllerImageTag, - Message: DisabledMsg, + Version: metalLBControllerImage.Tag, + Message: metallb.DisabledMsg, }, nil } @@ -42,7 +43,7 @@ func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadb err = fmt.Errorf("failed to enable LoadBalancer: %w", err) return types.FeatureStatus{ Enabled: false, - Version: ControllerImageTag, + Version: metalLBControllerImage.Tag, Message: fmt.Sprintf(deployFailedMsgTmpl, err), }, err } @@ -51,62 +52,47 @@ func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadb case loadbalancer.GetBGPMode(): return types.FeatureStatus{ Enabled: true, - Version: ControllerImageTag, - Message: fmt.Sprintf(enabledMsgTmpl, "BGP"), + Version: metalLBControllerImage.Tag, + Message: fmt.Sprintf(metallb.EnabledMsgTmpl, "BGP"), }, nil case loadbalancer.GetL2Mode(): return types.FeatureStatus{ Enabled: true, - Version: ControllerImageTag, - Message: fmt.Sprintf(enabledMsgTmpl, "L2"), + Version: metalLBControllerImage.Tag, + Message: fmt.Sprintf(metallb.EnabledMsgTmpl, "L2"), }, nil default: return types.FeatureStatus{ Enabled: true, - Version: ControllerImageTag, - Message: fmt.Sprintf(enabledMsgTmpl, "Unknown"), + Version: metalLBControllerImage.Tag, + Message: fmt.Sprintf(metallb.EnabledMsgTmpl, "Unknown"), }, nil } } func disableLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, network types.Network) error { - if _, err := m.Apply(ctx, ChartMetalLBLoadBalancer, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureLoadBalancer.GetChart(LoadBalancerChartName), helm.StateDeleted, nil); err != nil { return fmt.Errorf("failed to uninstall MetalLB LoadBalancer chart: %w", err) } - if _, err := m.Apply(ctx, ChartMetalLB, helm.StateDeleted, nil); err != nil { + if _, err := m.Apply(ctx, FeatureLoadBalancer.GetChart(MetalLBChartName), helm.StateDeleted, nil); err != nil { return fmt.Errorf("failed to uninstall MetalLB chart: %w", err) } return nil } func enableLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, loadbalancer types.LoadBalancer, network types.Network) error { - metalLBValues := map[string]any{ - "controller": map[string]any{ - "image": map[string]any{ - "repository": controllerImageRepo, - "tag": ControllerImageTag, - }, - "command": "/controller", - }, - "speaker": map[string]any{ - "image": map[string]any{ - "repository": speakerImageRepo, - "tag": speakerImageTag, - }, - "command": "/speaker", - // TODO(neoaggelos): make frr enable/disable configurable through an annotation - // We keep it disabled by default - "frr": map[string]any{ - "enabled": false, - "image": map[string]any{ - "repository": frrImageRepo, - "tag": frrImageTag, - }, - }, - }, + var metalLBValues MetalLBValues = map[string]any{} + + if err := metalLBValues.ApplyDefaultValues(); err != nil { + return fmt.Errorf("failed to apply default values: %w", err) + } + + if err := metalLBValues.ApplyImageOverrides(); err != nil { + return fmt.Errorf("failed to apply image overrides: %w", err) } - if _, err := m.Apply(ctx, ChartMetalLB, helm.StatePresent, metalLBValues); err != nil { + + if _, err := m.Apply(ctx, FeatureLoadBalancer.GetChart(MetalLBChartName), helm.StatePresent, metalLBValues); err != nil { return fmt.Errorf("failed to apply MetalLB configuration: %w", err) } @@ -114,37 +100,17 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, m helm.Client, load return fmt.Errorf("failed to wait for required MetalLB CRDs: %w", err) } - cidrs := []map[string]any{} - for _, cidr := range loadbalancer.GetCIDRs() { - cidrs = append(cidrs, map[string]any{"cidr": cidr}) - } - for _, ipRange := range loadbalancer.GetIPRanges() { - cidrs = append(cidrs, map[string]any{"start": ipRange.Start, "stop": ipRange.Stop}) + var values Values = map[string]any{} + + if err := values.ApplyDefaultValues(); err != nil { + return fmt.Errorf("failed to apply default values: %w", err) } - values := map[string]any{ - "driver": "metallb", - "l2": map[string]any{ - "enabled": loadbalancer.GetL2Mode(), - "interfaces": loadbalancer.GetL2Interfaces(), - }, - "ipPool": map[string]any{ - "cidrs": cidrs, - }, - "bgp": map[string]any{ - "enabled": loadbalancer.GetBGPMode(), - "localASN": loadbalancer.GetBGPLocalASN(), - "neighbors": []map[string]any{ - { - "peerAddress": loadbalancer.GetBGPPeerAddress(), - "peerASN": loadbalancer.GetBGPPeerASN(), - "peerPort": loadbalancer.GetBGPPeerPort(), - }, - }, - }, + if err := values.ApplyClusterConfiguration(loadbalancer); err != nil { + return fmt.Errorf("failed to apply cluster configuration: %w", err) } - if _, err := m.Apply(ctx, ChartMetalLBLoadBalancer, helm.StatePresent, values); err != nil { + if _, err := m.Apply(ctx, FeatureLoadBalancer.GetChart(LoadBalancerChartName), helm.StatePresent, values); err != nil { return fmt.Errorf("failed to apply MetalLB LoadBalancer configuration: %w", err) } diff --git a/src/k8s/pkg/k8sd/features/metallb/loadbalancer_test.go b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/loadbalancer_test.go similarity index 75% rename from src/k8s/pkg/k8sd/features/metallb/loadbalancer_test.go rename to src/k8s/pkg/k8sd/features/metallb/loadbalancer/loadbalancer_test.go index 04861151d..c1a9aefe3 100644 --- a/src/k8s/pkg/k8sd/features/metallb/loadbalancer_test.go +++ b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/loadbalancer_test.go @@ -1,4 +1,4 @@ -package metallb_test +package loadbalancer_test import ( "context" @@ -10,6 +10,7 @@ import ( helmmock "github.com/canonical/k8s/pkg/client/helm/mock" "github.com/canonical/k8s/pkg/client/kubernetes" "github.com/canonical/k8s/pkg/k8sd/features/metallb" + metallb_loadbalancer "github.com/canonical/k8s/pkg/k8sd/features/metallb/loadbalancer" "github.com/canonical/k8s/pkg/k8sd/types" snapmock "github.com/canonical/k8s/pkg/snap/mock" . "github.com/onsi/gomega" @@ -37,16 +38,16 @@ func TestDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&metallb.ChartFS)) - status, err := metallb.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) + status, err := metallb_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) - g.Expect(status.Version).To(Equal(metallb.ControllerImageTag)) + g.Expect(status.Version).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetImage(metallb_loadbalancer.MetalLBControllerImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(metallb.ChartMetalLBLoadBalancer)) + g.Expect(callArgs.Chart).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetChart(metallb_loadbalancer.LoadBalancerChartName))) g.Expect(callArgs.State).To(Equal(helm.StateDeleted)) g.Expect(callArgs.Values).To(BeNil()) }) @@ -64,21 +65,21 @@ func TestDisabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&metallb.ChartFS)) - status, err := metallb.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) + status, err := metallb_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(Equal(metallb.DisabledMsg)) - g.Expect(status.Version).To(Equal(metallb.ControllerImageTag)) + g.Expect(status.Version).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetImage(metallb_loadbalancer.MetalLBControllerImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(2)) firstCallArgs := helmM.ApplyCalledWith[0] - g.Expect(firstCallArgs.Chart).To(Equal(metallb.ChartMetalLBLoadBalancer)) + g.Expect(firstCallArgs.Chart).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetChart(metallb_loadbalancer.LoadBalancerChartName))) g.Expect(firstCallArgs.State).To(Equal(helm.StateDeleted)) g.Expect(firstCallArgs.Values).To(BeNil()) secondCallArgs := helmM.ApplyCalledWith[1] - g.Expect(secondCallArgs.Chart).To(Equal(metallb.ChartMetalLB)) + g.Expect(secondCallArgs.Chart).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetChart(metallb_loadbalancer.MetalLBChartName))) g.Expect(secondCallArgs.State).To(Equal(helm.StateDeleted)) g.Expect(secondCallArgs.Values).To(BeNil()) }) @@ -102,16 +103,16 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&metallb.ChartFS)) - status, err := metallb.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) + status, err := metallb_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) g.Expect(err).To(MatchError(applyErr)) g.Expect(status.Enabled).To(BeFalse()) g.Expect(status.Message).To(ContainSubstring(applyErr.Error())) - g.Expect(status.Version).To(Equal(metallb.ControllerImageTag)) + g.Expect(status.Version).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetImage(metallb_loadbalancer.MetalLBControllerImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(1)) callArgs := helmM.ApplyCalledWith[0] - g.Expect(callArgs.Chart).To(Equal(metallb.ChartMetalLB)) + g.Expect(callArgs.Chart).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetChart(metallb_loadbalancer.MetalLBChartName))) g.Expect(callArgs.State).To(Equal(helm.StatePresent)) // we don't validate values since it's just a static struct // and won't be changed by configurations @@ -165,22 +166,22 @@ func TestEnabled(t *testing.T) { } mc := snapM.HelmClient(loader.NewEmbedLoader(&metallb.ChartFS)) - status, err := metallb.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) + status, err := metallb_loadbalancer.ApplyLoadBalancer(context.Background(), snapM, mc, lbCfg, types.Network{}, nil) g.Expect(err).ToNot(HaveOccurred()) g.Expect(status.Enabled).To(BeTrue()) - g.Expect(status.Version).To(Equal(metallb.ControllerImageTag)) + g.Expect(status.Version).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetImage(metallb_loadbalancer.MetalLBControllerImageName).Tag)) g.Expect(helmM.ApplyCalledWith).To(HaveLen(2)) firstCallArgs := helmM.ApplyCalledWith[0] - g.Expect(firstCallArgs.Chart).To(Equal(metallb.ChartMetalLB)) + g.Expect(firstCallArgs.Chart).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetChart(metallb_loadbalancer.MetalLBChartName))) g.Expect(firstCallArgs.State).To(Equal(helm.StatePresent)) // we don't validate values since it's just a static struct // and won't be changed by configurations g.Expect(firstCallArgs.Values).ToNot(BeNil()) secondCallArgs := helmM.ApplyCalledWith[1] - g.Expect(secondCallArgs.Chart).To(Equal(metallb.ChartMetalLBLoadBalancer)) + g.Expect(secondCallArgs.Chart).To(Equal(metallb_loadbalancer.FeatureLoadBalancer.GetChart(metallb_loadbalancer.LoadBalancerChartName))) g.Expect(secondCallArgs.State).To(Equal(helm.StatePresent)) validateLoadBalancerValues(g, secondCallArgs.Values, lbCfg) }) diff --git a/src/k8s/pkg/k8sd/features/metallb/loadbalancer/manifest.go b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/manifest.go new file mode 100644 index 000000000..20b093f99 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/manifest.go @@ -0,0 +1,54 @@ +package loadbalancer + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + MetalLBChartName = "metallb" + LoadBalancerChartName = "ck-loadbalancer" + + MetalLBControllerImageName = "metallb-controller" + MetalLBSpeakerImageName = "metallb-speaker" + FRRImageName = "frr" +) + +var manifest = types.FeatureManifest{ + Name: "loadbalancer", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + MetalLBChartName: { + Name: "metallb", + Version: "0.14.9", + InstallName: "metallb", + InstallNamespace: "metallb-system", + }, + LoadBalancerChartName: { + Name: "ck-loadbalancer", + Version: "0.1.1", + InstallName: "metallb-loadbalancer", + InstallNamespace: "metallb-system", + }, + }, + + Images: map[string]types.Image{ + MetalLBControllerImageName: { + Registry: "ghcr.io/canonical", + Repository: "metallb-controller", + Tag: "v0.14.9-ck0", + }, + MetalLBSpeakerImageName: { + Registry: "ghcr.io/canonical", + Repository: "metallb-speaker", + Tag: "v0.14.9-ck0", + }, + FRRImageName: { + Registry: "ghcr.io/canonical", + Repository: "frr", + Tag: "9.1.3", + }, + }, +} + +var FeatureLoadBalancer types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/metallb/loadbalancer/register.go b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/register.go new file mode 100644 index 000000000..8f2d0bece --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/register.go @@ -0,0 +1,19 @@ +package loadbalancer + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + metalLBControllerImage := FeatureLoadBalancer.GetImage(MetalLBControllerImageName) + metalLBSpeakerImage := FeatureLoadBalancer.GetImage(MetalLBSpeakerImageName) + frrImage := FeatureLoadBalancer.GetImage(FRRImageName) + + images.Register( + fmt.Sprintf("%s:%s", metalLBControllerImage.GetURI(), metalLBControllerImage.Tag), + fmt.Sprintf("%s:%s", metalLBSpeakerImage.GetURI(), metalLBSpeakerImage.Tag), + fmt.Sprintf("%s:%s", frrImage.GetURI(), frrImage.Tag), + ) +} diff --git a/src/k8s/pkg/k8sd/features/metallb/loadbalancer/values.go b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/values.go new file mode 100644 index 000000000..ce80d1fb0 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/loadbalancer/values.go @@ -0,0 +1,116 @@ +package loadbalancer + +import ( + "fmt" + + "dario.cat/mergo" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +type MetalLBValues map[string]interface{} + +func (v MetalLBValues) ApplyDefaultValues() error { + values := map[string]any{ + "controller": map[string]any{ + "command": "/controller", + }, + "speaker": map[string]any{ + "command": "/speaker", + // TODO(neoaggelos): make frr enable/disable configurable through an annotation + // We keep it disabled by default + "frr": map[string]any{ + "enabled": false, + }, + }, + } + + if err := mergo.Merge(&v, MetalLBValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v MetalLBValues) ApplyImageOverrides() error { + metalLBControllerImage := FeatureLoadBalancer.GetImage(MetalLBControllerImageName) + metalLBSpeakerImage := FeatureLoadBalancer.GetImage(MetalLBSpeakerImageName) + frrImage := FeatureLoadBalancer.GetImage(FRRImageName) + + values := map[string]any{ + "controller": map[string]any{ + "image": map[string]any{ + "repository": metalLBControllerImage.GetURI(), + "tag": metalLBControllerImage.Tag, + }, + }, + "speaker": map[string]any{ + "image": map[string]any{ + "repository": metalLBSpeakerImage.GetURI(), + "tag": metalLBSpeakerImage.Tag, + }, + "frr": map[string]any{ + "image": map[string]any{ + "repository": frrImage.GetURI(), + "tag": frrImage.Tag, + }, + }, + }, + } + + if err := mergo.Merge(&v, MetalLBValues(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +type Values map[string]any + +func (v Values) ApplyDefaultValues() error { + values := map[string]any{ + "driver": "metallb", + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyClusterConfiguration(loadbalancer types.LoadBalancer) error { + cidrs := []map[string]any{} + for _, cidr := range loadbalancer.GetCIDRs() { + cidrs = append(cidrs, map[string]any{"cidr": cidr}) + } + for _, ipRange := range loadbalancer.GetIPRanges() { + cidrs = append(cidrs, map[string]any{"start": ipRange.Start, "stop": ipRange.Stop}) + } + + values := map[string]any{ + "l2": map[string]any{ + "enabled": loadbalancer.GetL2Mode(), + "interfaces": loadbalancer.GetL2Interfaces(), + }, + "ipPool": map[string]any{ + "cidrs": cidrs, + }, + "bgp": map[string]any{ + "enabled": loadbalancer.GetBGPMode(), + "localASN": loadbalancer.GetBGPLocalASN(), + "neighbors": []map[string]any{ + { + "peerAddress": loadbalancer.GetBGPPeerAddress(), + "peerASN": loadbalancer.GetBGPPeerASN(), + "peerPort": loadbalancer.GetBGPPeerPort(), + }, + }, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge cluster configuration values: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/features/metallb/register.go b/src/k8s/pkg/k8sd/features/metallb/register.go index 77a5e21b5..35fd75838 100644 --- a/src/k8s/pkg/k8sd/features/metallb/register.go +++ b/src/k8s/pkg/k8sd/features/metallb/register.go @@ -1,18 +1,9 @@ package metallb import ( - "fmt" - "github.com/canonical/k8s/pkg/k8sd/charts" - "github.com/canonical/k8s/pkg/k8sd/images" ) func init() { - images.Register( - fmt.Sprintf("%s:%s", controllerImageRepo, ControllerImageTag), - fmt.Sprintf("%s:%s", speakerImageRepo, speakerImageTag), - fmt.Sprintf("%s:%s", frrImageRepo, frrImageTag), - ) - charts.Register(&ChartFS) } diff --git a/src/k8s/pkg/k8sd/features/metallb/status.go b/src/k8s/pkg/k8sd/features/metallb/status.go new file mode 100644 index 000000000..4c60a709c --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/status.go @@ -0,0 +1,6 @@ +package metallb + +const ( + EnabledMsgTmpl = "enabled, %s mode" + DisabledMsg = "disabled" +) diff --git a/src/k8s/pkg/k8sd/features/metrics-server/chart.go b/src/k8s/pkg/k8sd/features/metrics-server/chart.go index 4e5fd8e34..a5f2275ef 100644 --- a/src/k8s/pkg/k8sd/features/metrics-server/chart.go +++ b/src/k8s/pkg/k8sd/features/metrics-server/chart.go @@ -2,25 +2,7 @@ package metrics_server import ( "embed" - - "github.com/canonical/k8s/pkg/client/helm" ) //go:embed all:charts var ChartFS embed.FS - -var ( - // chart represents manifests to deploy metrics-server. - chart = helm.InstallableChart{ - Name: "metrics-server", - Version: "3.12.2", - InstallName: "metrics-server", - InstallNamespace: "kube-system", - } - - // imageRepo is the image to use for metrics-server. - imageRepo = "ghcr.io/canonical/metrics-server" - - // imageTag is the image tag to use for metrics-server. - imageTag = "0.7.2-ck0" -) diff --git a/src/k8s/pkg/k8sd/features/metrics-server/internal.go b/src/k8s/pkg/k8sd/features/metrics-server/internal.go index c927e348d..992ee97e4 100644 --- a/src/k8s/pkg/k8sd/features/metrics-server/internal.go +++ b/src/k8s/pkg/k8sd/features/metrics-server/internal.go @@ -11,10 +11,7 @@ type config struct { } func internalConfig(annotations types.Annotations) config { - config := config{ - imageRepo: imageRepo, - imageTag: imageTag, - } + config := config{} if v, ok := annotations.Get(apiv1_annotations.AnnotationImageRepo); ok { config.imageRepo = v diff --git a/src/k8s/pkg/k8sd/features/metrics-server/manifest.go b/src/k8s/pkg/k8sd/features/metrics-server/manifest.go new file mode 100644 index 000000000..cac5a55dc --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metrics-server/manifest.go @@ -0,0 +1,35 @@ +package metrics_server + +import ( + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" +) + +var ( + MetricsServerChartName = "metrics-server" + + MetricsServerImageName = "metrics-server" +) + +var manifest = types.FeatureManifest{ + Name: "metrics-server", + Version: "1.0.0", + Charts: map[string]helm.InstallableChart{ + MetricsServerChartName: { + Name: "metrics-server", + Version: "3.12.2", + InstallName: "metrics-server", + InstallNamespace: "kube-system", + }, + }, + + Images: map[string]types.Image{ + MetricsServerImageName: { + Registry: "ghcr.io/canonical", + Repository: "metrics-server", + Tag: "0.7.2-ck0", + }, + }, +} + +var FeatureMetricsServer types.Feature = manifest diff --git a/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go b/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go index c02f9dca9..2fa6d04b6 100644 --- a/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go +++ b/src/k8s/pkg/k8sd/features/metrics-server/metrics_server.go @@ -23,20 +23,45 @@ const ( // ApplyMetricsServer returns an error if anything fails. The error is also wrapped in the .Message field of the // returned FeatureStatus. func ApplyMetricsServer(ctx context.Context, snap snap.Snap, m helm.Client, cfg types.MetricsServer, annotations types.Annotations) (types.FeatureStatus, error) { - config := internalConfig(annotations) + metricsServerImage := FeatureMetricsServer.GetImage(MetricsServerImageName) + imageTag := metricsServerImage.Tag - values := map[string]any{ - "image": map[string]any{ - "repository": config.imageRepo, - "tag": config.imageTag, - }, - "securityContext": map[string]any{ - // ROCKs with Pebble as the entrypoint do not work with this option. - "readOnlyRootFilesystem": false, - }, + config := config{} + + if config.imageTag != "" { + imageTag = config.imageTag + } + + var values Values = map[string]any{} + + if err := values.ApplyDefaultValues(); err != nil { + err = fmt.Errorf("failed to apply default values: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: imageTag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyImageOverrides(); err != nil { + err = fmt.Errorf("failed to apply image overrides: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: imageTag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err + } + + if err := values.ApplyAnnotations(annotations); err != nil { + err = fmt.Errorf("failed to apply annotations: %w", err) + return types.FeatureStatus{ + Enabled: false, + Version: imageTag, + Message: fmt.Sprintf(deployFailedMsgTmpl, err), + }, err } - _, err := m.Apply(ctx, chart, helm.StatePresentOrDeleted(cfg.GetEnabled()), values) + _, err := m.Apply(ctx, FeatureMetricsServer.GetChart(MetricsServerChartName), helm.StatePresentOrDeleted(cfg.GetEnabled()), values) if err != nil { if cfg.GetEnabled() { err = fmt.Errorf("failed to install metrics server chart: %w", err) diff --git a/src/k8s/pkg/k8sd/features/metrics-server/register.go b/src/k8s/pkg/k8sd/features/metrics-server/register.go index 9f4e4311c..43f5e4343 100644 --- a/src/k8s/pkg/k8sd/features/metrics-server/register.go +++ b/src/k8s/pkg/k8sd/features/metrics-server/register.go @@ -8,8 +8,10 @@ import ( ) func init() { + metricsServerImage := FeatureMetricsServer.GetImage(MetricsServerImageName) + images.Register( - fmt.Sprintf("%s:%s", imageRepo, imageTag), + fmt.Sprintf("%s:%s", metricsServerImage.GetURI(), metricsServerImage.Tag), ) charts.Register(&ChartFS) diff --git a/src/k8s/pkg/k8sd/features/metrics-server/values.go b/src/k8s/pkg/k8sd/features/metrics-server/values.go new file mode 100644 index 000000000..340e2166b --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metrics-server/values.go @@ -0,0 +1,65 @@ +package metrics_server + +import ( + "fmt" + + "dario.cat/mergo" +) + +type Values map[string]any + +func (v Values) ApplyDefaultValues() error { + values := map[string]any{ + "securityContext": map[string]any{ + // ROCKs with Pebble as the entrypoint do not work with this option. + "readOnlyRootFilesystem": false, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge default values: %w", err) + } + + return nil +} + +func (v Values) ApplyImageOverrides() error { + metricsServerImage := FeatureMetricsServer.GetImage(MetricsServerImageName) + + values := map[string]any{ + "image": map[string]any{ + "repository": metricsServerImage.GetURI(), + "tag": metricsServerImage.Tag, + }, + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge image overrides: %w", err) + } + + return nil +} + +func (v Values) ApplyAnnotations(annotations map[string]string) error { + config := internalConfig(annotations) + + image := map[string]any{} + + values := map[string]any{ + "image": image, + } + + if config.imageRepo != "" { + image["repository"] = config.imageRepo + } + + if config.imageTag != "" { + image["tag"] = config.imageTag + } + + if err := mergo.Merge(&v, Values(values), mergo.WithOverride); err != nil { + return fmt.Errorf("failed to merge annotation overrides: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/k8sd/types/feature.go b/src/k8s/pkg/k8sd/types/feature.go new file mode 100644 index 000000000..f92823978 --- /dev/null +++ b/src/k8s/pkg/k8sd/types/feature.go @@ -0,0 +1,53 @@ +package types + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/client/helm" +) + +type Image struct { + Registry string `yaml:"registry,omitempty"` + Repository string `yaml:"repository,omitempty"` + Tag string `yaml:"tag,omitempty"` +} + +func (i Image) GetURI() string { + if i.Registry == "" { + return i.Repository + } + + return fmt.Sprintf("%s/%s", i.Registry, i.Repository) +} + +type Feature interface { + GetName() string + GetVersion() string + GetChart(name string) helm.InstallableChart + GetImage(name string) Image +} + +type FeatureManifest struct { + Name string `yaml:"name,omitempty"` + Version string `yaml:"version,omitempty"` + + Charts map[string]helm.InstallableChart `yaml:"charts,omitempty"` + + Images map[string]Image `yaml:"images,omitempty"` +} + +func (f FeatureManifest) GetName() string { + return f.Name +} + +func (f FeatureManifest) GetVersion() string { + return f.Version +} + +func (f FeatureManifest) GetChart(name string) helm.InstallableChart { + return f.Charts[name] +} + +func (f FeatureManifest) GetImage(name string) Image { + return f.Images[name] +}