Skip to content

Commit 758af11

Browse files
glrfglrf
authored
Merge pull request #14 from appuio/feat/request-ratio-events
Add RatioController for detecting resource ratio violations
2 parents 7e802c0 + 79818f1 commit 758af11

File tree

16 files changed

+923
-127
lines changed

16 files changed

+923
-127
lines changed

‎.github/workflows/fuzz.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Fuzz
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- master
7+
push:
8+
branches:
9+
- master
10+
11+
jobs:
12+
fuzz:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v3
16+
17+
- name: Determine Go version from go.mod
18+
run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV
19+
20+
- uses: actions/setup-go@v3
21+
with:
22+
go-version: ${{ env.GO_VERSION }}
23+
24+
- uses: actions/cache@v2
25+
with:
26+
path: ~/go/pkg/mod
27+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
28+
restore-keys: |
29+
${{ runner.os }}-go-
30+
31+
- name: Run fuzz tests
32+
run: make fuzz

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,6 @@ node_modules/
1616

1717
# tem cert
1818
webhook-certs/
19+
20+
# fuzz testcases
21+
*/testdata/fuzz/

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ run:
3737
.PHONY: test
3838
test: test-go ## All-in-one test
3939

40+
.PHONY: fuzz
41+
fuzz:
42+
go test ./ratio -fuzztime 1m -fuzz .
43+
4044
.PHONY: test-go
4145
test-go: ## Run unit tests against code
4246
go test -race -coverprofile cover.out -covermode atomic ./...

‎PROJECT‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,8 @@ layout:
33
- go.kubebuilder.io/v3
44
projectName: appuio-cloud-agent
55
repo: github.com/appuio/appuio-cloud-agent
6+
resources:
7+
- controller: true
8+
kind: Pod
9+
version: v1
610
version: "3"

‎config/rbac/role.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ metadata:
55
creationTimestamp: null
66
name: appuio-cloud-agent
77
rules:
8+
- apiGroups:
9+
- ""
10+
resources:
11+
- events
12+
verbs:
13+
- create
14+
- patch
815
- apiGroups:
916
- ""
1017
resources:

‎controllers/ratio_controller.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
"errors"
6+
7+
"github.com/appuio/appuio-cloud-agent/ratio"
8+
corev1 "k8s.io/api/core/v1"
9+
"k8s.io/client-go/tools/record"
10+
11+
"k8s.io/apimachinery/pkg/api/resource"
12+
"k8s.io/apimachinery/pkg/runtime"
13+
ctrl "sigs.k8s.io/controller-runtime"
14+
"sigs.k8s.io/controller-runtime/pkg/client"
15+
"sigs.k8s.io/controller-runtime/pkg/log"
16+
)
17+
18+
// RatioReconciler reconciles a Pod object
19+
type RatioReconciler struct {
20+
client.Client
21+
Recorder record.EventRecorder
22+
Scheme *runtime.Scheme
23+
24+
Ratio ratioFetcher
25+
RatioLimit *resource.Quantity
26+
}
27+
28+
type ratioFetcher interface {
29+
FetchRatio(ctx context.Context, ns string) (*ratio.Ratio, error)
30+
}
31+
32+
//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
33+
//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch
34+
35+
var eventReason = "TooMuchCPURequest"
36+
37+
// Reconcile reacts to pod updates and emits events if the fair use request ratio is violated
38+
func (r *RatioReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
39+
l := log.FromContext(ctx).WithValues("namespace", req.Namespace, "name", req.Name)
40+
41+
nsRatio, err := r.Ratio.FetchRatio(ctx, req.Namespace)
42+
if err != nil {
43+
if errors.Is(err, ratio.ErrorDisabled) {
44+
l.V(1).Info("namespace disabled")
45+
return ctrl.Result{}, nil
46+
}
47+
l.Error(err, "failed to get ratio")
48+
return ctrl.Result{}, err
49+
}
50+
51+
if nsRatio.Below(*r.RatioLimit) {
52+
l.Info("recording warn event: ratio too low")
53+
54+
if err := r.warnPod(ctx, req.Name, req.Namespace, nsRatio); err != nil {
55+
l.Error(err, "failed to record event on pod")
56+
}
57+
if err := r.warnNamespace(ctx, req.Namespace, nsRatio); err != nil {
58+
l.Error(err, "failed to record event on namespace")
59+
}
60+
}
61+
62+
return ctrl.Result{}, nil
63+
}
64+
65+
func (r *RatioReconciler) warnPod(ctx context.Context, name, namespace string, nsRatio *ratio.Ratio) error {
66+
pod := corev1.Pod{}
67+
err := r.Get(ctx, client.ObjectKey{
68+
Namespace: namespace,
69+
Name: name,
70+
}, &pod)
71+
if err != nil {
72+
return err
73+
}
74+
r.Recorder.Event(&pod, "Warning", eventReason, nsRatio.Warn(r.RatioLimit))
75+
return nil
76+
}
77+
func (r *RatioReconciler) warnNamespace(ctx context.Context, name string, nsRatio *ratio.Ratio) error {
78+
ns := corev1.Namespace{}
79+
err := r.Get(ctx, client.ObjectKey{
80+
Name: name,
81+
}, &ns)
82+
if err != nil {
83+
return err
84+
}
85+
r.Recorder.Event(&ns, "Warning", eventReason, nsRatio.Warn(r.RatioLimit))
86+
return nil
87+
}
88+
89+
// SetupWithManager sets up the controller with the Manager.
90+
func (r *RatioReconciler) SetupWithManager(mgr ctrl.Manager) error {
91+
return ctrl.NewControllerManagedBy(mgr).
92+
For(&corev1.Pod{}).
93+
Complete(r)
94+
}

‎controllers/ratio_controller_test.go‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
"errors"
6+
7+
"github.com/appuio/appuio-cloud-agent/ratio"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
corev1 "k8s.io/api/core/v1"
11+
"k8s.io/apimachinery/pkg/api/resource"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/apimachinery/pkg/runtime"
14+
"k8s.io/apimachinery/pkg/types"
15+
ctrl "sigs.k8s.io/controller-runtime"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
18+
"testing"
19+
20+
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
21+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
22+
"k8s.io/client-go/tools/record"
23+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
24+
)
25+
26+
func TestRatioReconciler_Warn(t *testing.T) {
27+
recorder := record.NewFakeRecorder(4)
28+
_, err := prepareTest(t, testCfg{
29+
limit: resource.MustParse("4G"),
30+
fetchMemory: resource.MustParse("4G"),
31+
fetchCPU: resource.MustParse("1100m"),
32+
recorder: recorder,
33+
obj: []client.Object{
34+
testNs,
35+
testPod,
36+
},
37+
}).Reconcile(context.TODO(), ctrl.Request{
38+
NamespacedName: types.NamespacedName{
39+
Namespace: testNs.Name,
40+
Name: testPod.Name,
41+
},
42+
})
43+
assert.NoError(t, err)
44+
require.Len(t, recorder.Events, 2)
45+
}
46+
47+
func TestRatioReconciler_Ok(t *testing.T) {
48+
recorder := record.NewFakeRecorder(4)
49+
_, err := prepareTest(t, testCfg{
50+
limit: resource.MustParse("4G"),
51+
fetchMemory: resource.MustParse("4G"),
52+
fetchCPU: resource.MustParse("900m"),
53+
recorder: recorder,
54+
obj: []client.Object{
55+
testNs,
56+
testPod,
57+
},
58+
}).Reconcile(context.TODO(), ctrl.Request{
59+
NamespacedName: types.NamespacedName{
60+
Namespace: testNs.Name,
61+
Name: testPod.Name,
62+
},
63+
})
64+
assert.NoError(t, err)
65+
require.Len(t, recorder.Events, 0)
66+
}
67+
68+
func TestRatioReconciler_Disabled(t *testing.T) {
69+
recorder := record.NewFakeRecorder(4)
70+
_, err := prepareTest(t, testCfg{
71+
limit: resource.MustParse("4G"),
72+
fetchErr: ratio.ErrorDisabled,
73+
recorder: recorder,
74+
obj: []client.Object{
75+
testNs,
76+
testPod,
77+
},
78+
}).Reconcile(context.TODO(), ctrl.Request{
79+
NamespacedName: types.NamespacedName{
80+
Namespace: testNs.Name,
81+
Name: testPod.Name,
82+
},
83+
})
84+
assert.NoError(t, err)
85+
require.Len(t, recorder.Events, 0)
86+
}
87+
88+
func TestRatioReconciler_Failed(t *testing.T) {
89+
recorder := record.NewFakeRecorder(4)
90+
_, err := prepareTest(t, testCfg{
91+
limit: resource.MustParse("4G"),
92+
fetchErr: errors.New("internal"),
93+
recorder: recorder,
94+
obj: []client.Object{
95+
testNs,
96+
testPod,
97+
},
98+
}).Reconcile(context.TODO(), ctrl.Request{
99+
NamespacedName: types.NamespacedName{
100+
Namespace: testNs.Name,
101+
Name: testPod.Name,
102+
},
103+
})
104+
assert.Error(t, err)
105+
require.Len(t, recorder.Events, 0)
106+
}
107+
108+
func TestRatioReconciler_RecordFailed(t *testing.T) {
109+
wrongNs := *testNs
110+
wrongNs.Name = "bar"
111+
wrongPod := *testPod
112+
wrongPod.Name = "asf"
113+
wrongPod.Namespace = "asf"
114+
recorder := record.NewFakeRecorder(4)
115+
_, err := prepareTest(t, testCfg{
116+
limit: resource.MustParse("4G"),
117+
fetchMemory: resource.MustParse("4G"),
118+
fetchCPU: resource.MustParse("1100m"),
119+
recorder: recorder,
120+
obj: []client.Object{
121+
&wrongNs,
122+
&wrongPod,
123+
},
124+
}).Reconcile(context.TODO(), ctrl.Request{
125+
NamespacedName: types.NamespacedName{
126+
Namespace: testNs.Name,
127+
Name: testPod.Name,
128+
},
129+
})
130+
assert.NoError(t, err)
131+
if !assert.Len(t, recorder.Events, 0) {
132+
for i := 0; i < len(recorder.Events); i++ {
133+
e := <-recorder.Events
134+
t.Log(e)
135+
}
136+
}
137+
}
138+
139+
type testCfg struct {
140+
limit resource.Quantity
141+
fetchErr error
142+
fetchCPU resource.Quantity
143+
fetchMemory resource.Quantity
144+
obj []client.Object
145+
recorder record.EventRecorder
146+
}
147+
148+
func prepareTest(t *testing.T, cfg testCfg) *RatioReconciler {
149+
scheme := runtime.NewScheme()
150+
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
151+
152+
client := fake.NewClientBuilder().
153+
WithScheme(scheme).
154+
WithObjects(cfg.obj...).
155+
Build()
156+
157+
if cfg.recorder == nil {
158+
cfg.recorder = &record.FakeRecorder{}
159+
}
160+
161+
return &RatioReconciler{
162+
Client: client,
163+
Recorder: cfg.recorder,
164+
Scheme: scheme,
165+
Ratio: fakeFetcher{
166+
err: cfg.fetchErr,
167+
ratio: &ratio.Ratio{
168+
CPU: cfg.fetchCPU.AsDec(),
169+
Memory: cfg.fetchMemory.AsDec(),
170+
},
171+
},
172+
RatioLimit: &cfg.limit,
173+
}
174+
}
175+
176+
type fakeFetcher struct {
177+
err error
178+
ratio *ratio.Ratio
179+
}
180+
181+
func (f fakeFetcher) FetchRatio(ctx context.Context, ns string) (*ratio.Ratio, error) {
182+
return f.ratio, f.err
183+
}
184+
185+
var testNs = &corev1.Namespace{
186+
ObjectMeta: metav1.ObjectMeta{
187+
Name: "foo",
188+
},
189+
}
190+
var testPod = &corev1.Pod{
191+
ObjectMeta: metav1.ObjectMeta{
192+
Name: "pod",
193+
Namespace: "foo",
194+
},
195+
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.18
44

55
require (
66
github.com/stretchr/testify v1.7.1
7+
gopkg.in/inf.v0 v0.9.1
78
k8s.io/api v0.23.5
89
k8s.io/apimachinery v0.23.5
910
k8s.io/client-go v0.23.5
@@ -64,7 +65,6 @@ require (
6465
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
6566
google.golang.org/appengine v1.6.7 // indirect
6667
google.golang.org/protobuf v1.27.1 // indirect
67-
gopkg.in/inf.v0 v0.9.1 // indirect
6868
gopkg.in/yaml.v2 v2.4.0 // indirect
6969
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
7070
k8s.io/apiextensions-apiserver v0.23.5 // indirect

0 commit comments

Comments
 (0)