Expose server gossip and RPC ports as hostPorts

To enable a client agent outside of the k8s cluster to join the datacenter, you would need to enable server.exposeGossipAndRPCPorts, client.exposeGossipPorts, and set server.serflan.port to a port not being used on the host. Since client.exposeGossipPorts uses the hostPort 8301, server.serflan.port must be set to something other than 8301. The client agent VM outside of the k8s cluster would need to be able to route to the private IP of the VMs in the k8s cluster to join the datacenter and the VMs in the k8s cluster would need to be able to route to the client agent VM outside the k8s cluster as well on its advertised IP.
hashicorp · Dec 17, 2020 · 6031a41 · 6031a41
1 parent 1c9cf18
commit 6031a41
Show file tree

Hide file tree

Showing 5 changed files with 206 additions and 17 deletions.
diff --git a/templates/client-daemonset.yaml b/templates/client-daemonset.yaml
@@ -121,12 +121,12 @@ spec:
             - name: ADVERTISE_IP
               valueFrom:
                 fieldRef:
-                  {{- if not .Values.client.exposeGossipPorts }}
-                  fieldPath: status.podIP
-                  {{- else }}
-                  # Clients will be exposed on their node's hostPort for external-to-k8s communication,
-                  # so they need to advertise their host ip instead of their pod ip.
+                  {{- if .Values.client.exposeGossipPorts }}
+                  {{- /* Clients will be exposed on their node's hostPort for external-to-k8s communication,
+                  so they need to advertise their host ip instead of their pod ip. */}}
                   fieldPath: status.hostIP
+                  {{- else }}
+                  fieldPath: status.podIP
                   {{- end }}
             - name: NAMESPACE
               valueFrom:
@@ -219,8 +219,9 @@ spec:
                 {{- end }}
                 {{- else }}
                 {{- if .Values.server.enabled }}
+                {{- $serverSerfLANPort  := .Values.server.serflan.port -}}
                 {{- range $index := until (.Values.server.replicas | int) }}
-                -retry-join="${CONSUL_FULLNAME}-server-{{ $index }}.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc" \
+                -retry-join="${CONSUL_FULLNAME}-server-{{ $index }}.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:{{ $serverSerfLANPort }}" \
                 {{- end }}
                 {{- end }}
                 {{- end }}

diff --git a/templates/server-statefulset.yaml b/templates/server-statefulset.yaml
@@ -106,6 +106,18 @@ spec:
         - name: consul
           image: "{{ default .Values.global.image .Values.server.image }}"
           env:
+            - name: ADVERTISE_IP
+              valueFrom:
+                fieldRef:
+                  {{- if .Values.server.exposeGossipAndRPCPorts }}
+                  {{- /* Server gossip and RPC ports will be exposed as a hostPort
+                  on the hostIP, so they need to advertise their host ip
+                  instead of their pod ip. This is to support external client
+                  agents. */}}
+                  fieldPath: status.hostIP
+                  {{- else }}
+                  fieldPath: status.podIP
+                  {{- end }}
             - name: POD_IP
               valueFrom:
                 fieldRef:
@@ -142,7 +154,7 @@ spec:
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
               exec /bin/consul agent \
-                -advertise="${POD_IP}" \
+                -advertise="${ADVERTISE_IP}" \
                 -bind=0.0.0.0 \
                 -bootstrap-expect={{ if .Values.server.bootstrapExpect }}{{ .Values.server.bootstrapExpect }}{{ else }}{{ .Values.server.replicas }}{{ end }} \
                 {{- if .Values.global.tls.enabled }}
@@ -190,9 +202,11 @@ spec:
                 {{- if .Values.ui.enabled }}
                 -ui \
                 {{- end }}
+                {{- $serverSerfLANPort  := .Values.server.serflan.port -}}
                 {{- range $index := until (.Values.server.replicas | int) }}
-                -retry-join=${CONSUL_FULLNAME}-server-{{ $index }}.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc \
+                -retry-join="${CONSUL_FULLNAME}-server-{{ $index }}.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:{{ $serverSerfLANPort }}" \
                 {{- end }}
+                -serf-lan-port={{ .Values.server.serflan.port }} \
                 -server
           volumeMounts:
             - name: data-{{ .Release.Namespace }}
@@ -228,11 +242,24 @@ spec:
             - containerPort: 8501
               name: https
             {{- end }}
-            - containerPort: 8301
-              name: serflan
+            - containerPort: {{ .Values.server.serflan.port }}
+              {{- if .Values.server.exposeGossipAndRPCPorts }}
+              hostPort: {{ .Values.server.serflan.port }}
+              {{- end }}
+              protocol: "TCP"
+              name: serflan-tcp
+            - containerPort: {{ .Values.server.serflan.port }}
+              {{- if .Values.server.exposeGossipAndRPCPorts }}
+              hostPort: {{ .Values.server.serflan.port }}
+              {{- end }}
+              protocol: "UDP"
+              name: serflan-udp
             - containerPort: 8302
               name: serfwan
             - containerPort: 8300
+              {{- if .Values.server.exposeGossipAndRPCPorts }}
+              hostPort: 8300
+              {{- end }}
               name: server
             - containerPort: 8600
               name: dns-tcp

diff --git a/test/unit/client-daemonset.bats b/test/unit/client-daemonset.bats
@@ -79,13 +79,32 @@ load _helpers
       . | tee /dev/stderr |
       yq -r '.spec.template.spec.containers[0].command' | tee /dev/stderr)
 
-  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-0.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc\""))' | tee /dev/stderr)
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-0.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:8301\""))' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 
-  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-1.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc\""))' | tee /dev/stderr)
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-1.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:8301\""))' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 
-  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-2.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc\""))' | tee /dev/stderr)
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-2.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:8301\""))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "client/DaemonSet: retry join uses the server.serflan port" {
+  cd `chart_dir`
+  local command=$(helm template \
+      -s templates/client-daemonset.yaml  \
+      --set 'server.replicas=3' \
+      --set 'server.serflan.port=9301' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-0.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:9301\""))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-1.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:9301\""))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-2.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:9301\""))' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 }
 

diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats
@@ -160,6 +160,128 @@ load _helpers
   [ "${actual}" = "foo" ]
 }
 
+#--------------------------------------------------------------------
+# exposeGossipAndRPCPorts
+
+@test "server/StatefulSet: server gossip and RPC ports are not exposed by default" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/server-statefulset.yaml  \
+      . | tee /dev/stderr )
+
+  # Test that hostPort is not set for gossip ports
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-tcp")' | yq -r '.hostPort' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-udp")' | yq -r '.hostPort' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  # Test that hostPort is not set for rpc ports
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "server")' | yq -r '.hostPort' | tee /dev/stderr)
+  [ "${actual}" = "null" ]
+
+  # Test that ADVERTISE_IP is being set to podIP
+  local actual=$(echo "$object" |
+    yq -r -c '.spec.template.spec.containers[0].env | map(select(.name == "ADVERTISE_IP"))' | tee /dev/stderr)
+  [ "${actual}" = '[{"name":"ADVERTISE_IP","valueFrom":{"fieldRef":{"fieldPath":"status.podIP"}}}]' ]
+}
+
+@test "server/StatefulSet: server gossip and RPC ports can be exposed" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/server-statefulset.yaml  \
+      --set 'server.exposeGossipAndRPCPorts=true' \
+      . | tee /dev/stderr)
+
+  # Test that hostPort is set for gossip ports
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-tcp")' | yq -r '.hostPort' | tee /dev/stderr)
+  [ "${actual}" = "8301" ]
+
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-udp")' | yq -r '.hostPort' | tee /dev/stderr)
+  [ "${actual}" = "8301" ]
+
+  # Test that hostPort is set for rpc ports
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "server")' | yq -r '.hostPort' | tee /dev/stderr)
+  [ "${actual}" = "8300" ]
+
+  # Test that ADVERTISE_IP is being set to hostIP
+  local actual=$(echo "$object" |
+    yq -r -c '.spec.template.spec.containers[0].env | map(select(.name == "ADVERTISE_IP"))' | tee /dev/stderr)
+  [ "${actual}" = '[{"name":"ADVERTISE_IP","valueFrom":{"fieldRef":{"fieldPath":"status.hostIP"}}}]' ]
+
+}
+
+#--------------------------------------------------------------------
+# serflan
+
+@test "server/StatefulSet: server.serflan.port is set to 8301 by default" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/server-statefulset.yaml  \
+      . | tee /dev/stderr )
+
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-tcp")' | yq -r '.containerPort' | tee /dev/stderr)
+  [ "${actual}" = "8301" ]
+
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-udp")' | yq -r '.containerPort' | tee /dev/stderr)
+  [ "${actual}" = "8301" ]
+
+  local command=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-serf-lan-port=8301"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/StatefulSet: server.serflan.port can be customized" {
+  cd `chart_dir`
+  local object=$(helm template \
+      -s templates/server-statefulset.yaml  \
+      --set 'server.serflan.port=9301' \
+      . | tee /dev/stderr )
+
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-tcp")' | yq -r '.containerPort' | tee /dev/stderr)
+  [ "${actual}" = "9301" ]
+
+  local actual=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].ports[] | select(.name == "serflan-udp")' | yq -r '.containerPort' | tee /dev/stderr)
+  [ "${actual}" = "9301" ]
+
+  local command=$(echo "$object" |
+      yq -r '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-serf-lan-port=9301"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/StatefulSet: retry join uses the server.serflan port" {
+  cd `chart_dir`
+  local command=$(helm template \
+      -s templates/server-statefulset.yaml  \
+      --set 'server.replicas=3' \
+      --set 'server.serflan.port=9301' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-0.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:9301\""))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-1.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:9301\""))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo $command | jq -r ' . | any(contains("-retry-join=\"${CONSUL_FULLNAME}-server-2.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc:9301\""))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 #--------------------------------------------------------------------
 # extraVolumes
 
@@ -574,19 +696,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-      yq -r '.[2].name' | tee /dev/stderr)
+      yq -r '.[3].name' | tee /dev/stderr)
   [ "${actual}" = "custom_proxy" ]
 
   local actual=$(echo $object |
-      yq -r '.[2].value' | tee /dev/stderr)
+      yq -r '.[3].value' | tee /dev/stderr)
   [ "${actual}" = "fakeproxy" ]
 
   local actual=$(echo $object |
-      yq -r '.[3].name' | tee /dev/stderr)
+      yq -r '.[4].name' | tee /dev/stderr)
   [ "${actual}" = "no_proxy" ]
 
   local actual=$(echo $object |
-      yq -r '.[3].value' | tee /dev/stderr)
+      yq -r '.[4].value' | tee /dev/stderr)
   [ "${actual}" = "custom_no_proxy" ]
 }
 

diff --git a/values.yaml b/values.yaml
@@ -295,6 +295,26 @@ server:
     # The key within the Kubernetes secret that holds the enterprise license.
     secretKey: null
 
+  # Exposes the servers' gossip and RPC ports as hostPorts. To enable a client
+  # agent outside of the k8s cluster to join the datacenter, you would need to
+  # enable `server.exposeGossipAndRPCPorts`, `client.exposeGossipPorts`, and
+  # set `server.serflan.port` to a port not being used on the host. Since
+  # `client.exposeGossipPorts` uses the hostPort 8301, `server.serflan.port`
+  # must be set to something other than 8301.
+  exposeGossipAndRPCPorts: false
+
+  # Configures the LAN gossip port for the consul servers. However, if you
+  # choose to enable `server.exposeGossipAndRPCPorts` and
+  # `client.exposeGossipPorts`, that will configure the LAN gossip ports on the
+  # servers and clients to be hostPorts, so if you are running clients and
+  # servers in the same node the ports will conflict if they are both 8301.
+  # When you enable `server.exposeGossipAndRPCPorts` and
+  # `client.exposeGossipPorts`, you must change this from the default to an
+  # unused port on the host, e.g. 9301. By default it is 8301 and configured as
+  # a containerPort on the consul server Pods.
+  serflan:
+    port: 8301
+
   # This defines the disk size for configuring the
   # servers' StatefulSet storage. For dynamically provisioned storage classes, this is the
   # desired size. For manually defined persistent volumes, this should be set to