Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit c11c46d

Browse files
jiahuifk8s-publishing-bot
jiahuif
authored and
k8s-publishing-bot
committedAug 31, 2021
implement healthz
for controller managers. Kubernetes-commit: 15e0336de2fed898ad4744e1387dce1a613b2887
1 parent eca5ca7 commit c11c46d

File tree

4 files changed

+326
-0
lines changed

4 files changed

+326
-0
lines changed
 

‎pkg/healthz/handler.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package healthz
18+
19+
import (
20+
"net/http"
21+
"sync"
22+
23+
"k8s.io/apiserver/pkg/server/healthz"
24+
"k8s.io/apiserver/pkg/server/mux"
25+
)
26+
27+
// MutableHealthzHandler returns a http.Handler that handles "/healthz"
28+
// following the standard healthz mechanism.
29+
//
30+
// This handler can register health checks after its creation, which
31+
// is originally not allowed with standard healthz handler.
32+
type MutableHealthzHandler struct {
33+
// handler is the underlying handler that will be replaced every time
34+
// new checks are added.
35+
handler http.Handler
36+
// mutex is a RWMutex that allows concurrent health checks (read)
37+
// but disallow replacing the handler at the same time (write).
38+
mutex sync.RWMutex
39+
checks []healthz.HealthChecker
40+
}
41+
42+
func (h *MutableHealthzHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
43+
h.mutex.RLock()
44+
defer h.mutex.RUnlock()
45+
46+
h.handler.ServeHTTP(writer, request)
47+
}
48+
49+
// AddHealthChecker adds health check(s) to the handler.
50+
//
51+
// Every time this function is called, the handler have to be re-initiated.
52+
// It is advised to add as many checks at once as possible.
53+
func (h *MutableHealthzHandler) AddHealthChecker(checks ...healthz.HealthChecker) {
54+
h.mutex.Lock()
55+
defer h.mutex.Unlock()
56+
57+
h.checks = append(h.checks, checks...)
58+
newMux := mux.NewPathRecorderMux("healthz")
59+
healthz.InstallHandler(newMux, h.checks...)
60+
h.handler = newMux
61+
}
62+
63+
func NewMutableHealthzHandler(checks ...healthz.HealthChecker) *MutableHealthzHandler {
64+
h := &MutableHealthzHandler{}
65+
h.AddHealthChecker(checks...)
66+
67+
return h
68+
}

‎pkg/healthz/handler_test.go

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package healthz
18+
19+
import (
20+
"fmt"
21+
"net/http"
22+
"net/http/httptest"
23+
"sync/atomic"
24+
"testing"
25+
"time"
26+
27+
"k8s.io/apiserver/pkg/server/healthz"
28+
)
29+
30+
func TestMutableHealthzHandler(t *testing.T) {
31+
badChecker := healthz.NamedCheck("bad", func(r *http.Request) error {
32+
return fmt.Errorf("bad")
33+
})
34+
for _, tc := range []struct {
35+
name string
36+
checkBatches [][]healthz.HealthChecker
37+
appendBad bool // appends bad check after batches above, and see if it fails afterwards
38+
path string
39+
expectedBody string
40+
expectedStatus int
41+
}{
42+
{
43+
name: "empty",
44+
checkBatches: [][]healthz.HealthChecker{},
45+
path: "/healthz",
46+
expectedBody: "ok",
47+
expectedStatus: http.StatusOK,
48+
},
49+
{
50+
name: "good",
51+
checkBatches: [][]healthz.HealthChecker{
52+
{NamedPingChecker("good")},
53+
},
54+
path: "/healthz",
55+
expectedBody: "ok",
56+
expectedStatus: http.StatusOK,
57+
},
58+
{
59+
name: "good verbose", // verbose only applies for successful checks
60+
checkBatches: [][]healthz.HealthChecker{
61+
{NamedPingChecker("good")}, // batch 1: good
62+
},
63+
path: "/healthz?verbose=true",
64+
expectedBody: "[+]good ok\nhealthz check passed\n",
65+
expectedStatus: http.StatusOK,
66+
},
67+
{
68+
name: "good and bad, same batch",
69+
checkBatches: [][]healthz.HealthChecker{
70+
{NamedPingChecker("good"), badChecker}, // batch 1: good, bad
71+
},
72+
path: "/healthz",
73+
expectedBody: "[+]good ok\n[-]bad failed: reason withheld\nhealthz check failed\n",
74+
expectedStatus: http.StatusInternalServerError,
75+
},
76+
{
77+
name: "good and bad, two batches",
78+
checkBatches: [][]healthz.HealthChecker{
79+
{NamedPingChecker("good")}, // batch 1: good
80+
{badChecker}, // batch 2: bad
81+
},
82+
path: "/healthz",
83+
expectedBody: "[+]good ok\n[-]bad failed: reason withheld\nhealthz check failed\n",
84+
expectedStatus: http.StatusInternalServerError,
85+
},
86+
{
87+
name: "two checks and append bad",
88+
checkBatches: [][]healthz.HealthChecker{
89+
{NamedPingChecker("foo"), NamedPingChecker("bar")},
90+
},
91+
path: "/healthz",
92+
expectedBody: "ok",
93+
expectedStatus: http.StatusOK,
94+
appendBad: true,
95+
},
96+
{
97+
name: "subcheck",
98+
checkBatches: [][]healthz.HealthChecker{
99+
{NamedPingChecker("good")}, // batch 1: good
100+
{badChecker}, // batch 2: bad
101+
},
102+
path: "/healthz/good",
103+
expectedBody: "ok",
104+
expectedStatus: http.StatusOK,
105+
},
106+
} {
107+
t.Run(tc.name, func(t *testing.T) {
108+
h := NewMutableHealthzHandler()
109+
for _, batch := range tc.checkBatches {
110+
h.AddHealthChecker(batch...)
111+
}
112+
req, err := http.NewRequest("GET", fmt.Sprintf("https://example.com%v", tc.path), nil)
113+
if err != nil {
114+
t.Fatalf("unexpected error: %v", err)
115+
}
116+
w := httptest.NewRecorder()
117+
h.ServeHTTP(w, req)
118+
if w.Code != tc.expectedStatus {
119+
t.Errorf("unexpected status: expected %v, got %v", tc.expectedStatus, w.Result().StatusCode)
120+
}
121+
if w.Body.String() != tc.expectedBody {
122+
t.Errorf("unexpected body: expected %v, got %v", tc.expectedBody, w.Body.String())
123+
}
124+
if tc.appendBad {
125+
h.AddHealthChecker(badChecker)
126+
w := httptest.NewRecorder()
127+
h.ServeHTTP(w, req)
128+
// should fail
129+
if w.Code != http.StatusInternalServerError {
130+
t.Errorf("did not fail after adding bad checker")
131+
}
132+
}
133+
})
134+
}
135+
}
136+
137+
// TestConcurrentChecks tests that the handler would not block on concurrent healthz requests.
138+
func TestConcurrentChecks(t *testing.T) {
139+
const N = 5
140+
stopChan := make(chan interface{})
141+
defer close(stopChan) // always close no matter passing or not
142+
concurrentChan := make(chan interface{}, N)
143+
var concurrentCount int32
144+
pausingCheck := healthz.NamedCheck("pausing", func(r *http.Request) error {
145+
atomic.AddInt32(&concurrentCount, 1)
146+
concurrentChan <- nil
147+
<-stopChan
148+
return nil
149+
})
150+
151+
h := NewMutableHealthzHandler(pausingCheck)
152+
for i := 0; i < N; i++ {
153+
go func() {
154+
req, _ := http.NewRequest(http.MethodGet, "https://example.com/healthz", nil)
155+
w := httptest.NewRecorder()
156+
h.ServeHTTP(w, req)
157+
}()
158+
}
159+
160+
giveUp := time.After(1 * time.Second) // should take <1ms if passing
161+
for i := 0; i < N; i++ {
162+
select {
163+
case <-giveUp:
164+
t.Errorf("given up waiting for concurrent checks to start.")
165+
return
166+
case <-concurrentChan:
167+
continue
168+
}
169+
}
170+
171+
if concurrentCount != N {
172+
t.Errorf("expected %v concurrency, got %v", N, concurrentCount)
173+
}
174+
}

‎pkg/healthz/healthz.go

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package healthz
18+
19+
import (
20+
"net/http"
21+
22+
"k8s.io/apiserver/pkg/server/healthz"
23+
)
24+
25+
// NamedPingChecker returns a health check with given name
26+
// that returns no error when checked.
27+
func NamedPingChecker(name string) healthz.HealthChecker {
28+
return NamedHealthChecker(name, healthz.PingHealthz)
29+
}
30+
31+
// NamedHealthChecker creates a named health check from
32+
// an unnamed one.
33+
func NamedHealthChecker(name string, check UnnamedHealthChecker) healthz.HealthChecker {
34+
return healthz.NamedCheck(name, check.Check)
35+
}
36+
37+
// UnnamedHealthChecker is an unnamed healthz checker.
38+
// The name of the check can be set by the controller manager.
39+
type UnnamedHealthChecker interface {
40+
Check(req *http.Request) error
41+
}
42+
43+
var _ UnnamedHealthChecker = (healthz.HealthChecker)(nil)

‎pkg/healthz/healthz_test.go

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package healthz
18+
19+
import (
20+
"fmt"
21+
"net/http"
22+
"testing"
23+
)
24+
25+
type checkWithMessage struct {
26+
message string
27+
}
28+
29+
func (c *checkWithMessage) Check(_ *http.Request) error {
30+
return fmt.Errorf("%s", c.message)
31+
}
32+
33+
func TestNamedHealthChecker(t *testing.T) {
34+
named := NamedHealthChecker("foo", &checkWithMessage{message: "hello"})
35+
if named.Name() != "foo" {
36+
t.Errorf("expected: %v, got: %v", "foo", named.Name())
37+
}
38+
if err := named.Check(nil); err.Error() != "hello" {
39+
t.Errorf("expected: %v, got: %v", "hello", err.Error())
40+
}
41+
}

0 commit comments

Comments
 (0)