kapacity-manager v0.2.0

.github/workflows/ci.yaml

@@ -5,6 +5,7 @@ permissions:
   contents: read
 jobs:
   lint-test:
+    name: Lint and test
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

.github/workflows/release.yaml

@@ -7,6 +7,7 @@ permissions:
   contents: write
 jobs:
   release-charts:
+    name: Release charts
     if: ${{ contains(github.event.head_commit.message, '#release') }}
     runs-on: ubuntu-latest
     steps:

README.md

@@ -1,5 +1,6 @@
 # Kapacity Charts
 
 [![License](https://img.shields.io/github/license/traas-stack/kapacity-charts)](https://www.apache.org/licenses/LICENSE-2.0.html)
+[![Release](https://github.com/traas-stack/kapacity-charts/actions/workflows/release.yaml/badge.svg)](https://github.com/traas-stack/kapacity-charts/actions/workflows/release.yaml)
 
 This repository contains [Helm](https://helm.sh/) charts of [Kapacity](https://github.com/traas-stack/kapacity).

charts/kapacity-manager/Chart.yaml

@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: v0.1.0
+appVersion: v0.2.0

charts/kapacity-manager/templates/config.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kapacity-config
+  labels:
+    {{- include "kapacity-manager.labels" . | nindent 4 }}
+data:
+  prometheus-metrics-config.yaml: |
+{{ toYaml .Values.prometheus.metrics.config | indent 4 }}

charts/kapacity-manager/templates/deployment.yaml

@@ -37,13 +37,34 @@ spec:
         - --log-file-rotation-time={{ .Values.logFileRotationTime }}
         - --zap-stacktrace-level={{ .Values.zap.stacktraceLevel }}
         - --zap-log-level={{ .Values.zap.logLevel }}
-        - --health-probe-bind-address=:8081
         - --metrics-bind-address=:8080
+        - --health-probe-bind-address=:8081
         - --leader-elect
         - --reconcile-concurrency={{ .Values.reconcileConcurrency }}
-        - --metric-provider={{ .Values.metricsProvider }}
-        {{- if eq .Values.metricsProvider "prometheus" }}
-        - --prometheus-address={{ .Values.prometheus.address }}
+        - --object-label-selector={{ .Values.objectSelector.label }}
+        - --object-field-selector={{ .Values.objectSelector.field }}
+        - --serve-admission-webhooks={{ .Values.webhook.enabled }}
+        - --grpc-server-bind-address={{ .Values.grpcServer.enabled | ternary (print .Values.grpcServer.bindAddress ":" .Values.grpcServer.bindPort) "0" }}
+        - --metric-provider={{ .Values.metricProvider }}
+        {{- if eq .Values.metricProvider "prometheus" }}
+        {{- with .Values.prometheus }}
+        - --prometheus-address={{ .address }}
+        - --prometheus-auth-incluster={{ .auth.incluster }}
+        - --prometheus-auth-config={{ .auth.config }}
+        - --prometheus-metrics-config=/etc/kapacity/prometheus-metrics-config.yaml
+        - --prometheus-metrics-relist-interval={{ .metrics.relistInterval }}
+        {{- if .metrics.maxAge }}
+        - --prometheus-metrics-max-age={{ .metrics.maxAge }}
+        {{- end }}
+        {{- end }}
+        {{- end }}
+        - --algorithm-job-namespace={{ .Release.Namespace }}
+        {{- with .Values.algorithmJob }}
+        - --algorithm-job-default-service-account={{ .defaultServiceAccount }}
+        - --algorithm-job-default-metrics-server-addr={{ .defaultMetricsServerAddr }}
+        {{- with .defaultImage.horizontalPredictive }}
+        - --algorithm-job-default-image-horizontal-predictive={{ .registry }}/{{ .repository }}:{{ .tag }}
+        {{- end }}
         {{- end }}
         env:
         - name: TZ
@@ -54,12 +75,17 @@ spec:
         {{- end }}
         image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
-        {{- if .Values.webhook.enabled }}
         ports:
+        {{- if .Values.webhook.enabled }}
         - containerPort: 9443
           name: webhook-server
           protocol: TCP
         {{- end }}
+        {{- if .Values.grpcServer.enabled }}
+        - containerPort: {{ .Values.grpcServer.bindPort }}
+          name: grpc-server
+          protocol: TCP
+        {{- end }}
         livenessProbe:
           httpGet:
             path: /healthz
@@ -74,8 +100,10 @@ spec:
           periodSeconds: 10
         resources:
           {{- toYaml .Values.resources | nindent 12 }}
-        {{- if or .Values.logPath .Values.webhook.enabled }}
         volumeMounts:
+        - name: config
+          mountPath: /etc/kapacity
+          readOnly: true
         {{- if .Values.logPath }}
         - name: log
           mountPath: {{ .Values.logPath }}
@@ -85,7 +113,6 @@ spec:
           mountPath: /tmp/k8s-webhook-server/serving-certs
           readOnly: true
         {{- end }}
-        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -104,8 +131,10 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if or .Values.logPath .Values.webhook.enabled }}
       volumes:
+      - name: config
+        configMap:
+          name: kapacity-config
       {{- if .Values.logPath }}
       - name: log
         emptyDir: {}
@@ -116,4 +145,3 @@ spec:
           defaultMode: 420
           secretName: webhook-server-cert
       {{- end }}
-      {{- end }}

charts/kapacity-manager/templates/rbac.yaml

@@ -45,6 +45,14 @@ metadata:
   labels:
     {{- include "kapacity-manager.labels" . | nindent 4 }}
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -167,6 +175,17 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
 - apiGroups:
   - custom.metrics.k8s.io
   resources:
@@ -192,6 +211,47 @@ rules:
   - list
   - watch
 
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kapacity:algorithm-job
+  labels:
+    {{- include "kapacity-manager.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - '*'
+  resources:
+  - '*/scale'
+  verbs:
+  - get
+- apiGroups:
+  - autoscaling.kapacitystack.io
+  resources:
+  - horizontalportraits
+  verbs:
+  - get
+  - list
+  - watch
+
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -223,3 +283,19 @@ subjects:
 - kind: ServiceAccount
   name: {{ include "kapacity-manager.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kapacity:algorithm-job
+  labels:
+    {{- include "kapacity-manager.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kapacity:algorithm-job
+subjects:
+- kind: ServiceAccount
+  name: kapacity-algorithm-job
+  namespace: {{ .Release.Namespace }}

charts/kapacity-manager/templates/service.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.grpcServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: kapacity-grpc-service
+  labels:
+    {{- include "kapacity-manager.labels" . | nindent 4 }}
+spec:
+  ports:
+  - port: {{ .Values.grpcServer.bindPort }}
+    protocol: TCP
+    targetPort: {{ .Values.grpcServer.bindPort }}
+  selector:
+    {{- include "kapacity-manager.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/kapacity-manager/templates/serviceaccount.yaml

@@ -8,3 +8,15 @@ metadata:
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kapacity-algorithm-job
+  labels:
+    {{- include "kapacity-manager.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}

charts/kapacity-manager/values.yaml

@@ -12,15 +12,102 @@ image:
   tag: ""
 
 reconcileConcurrency: 16
-metricsProvider: prometheus
+objectSelector:
+  label: ""
+  field: ""
+
+grpcServer:
+  enabled: true
+  bindAddress: ""
+  bindPort: 9090
+
+metricProvider: prometheus
 # Setup Promethues provider configs.
 prometheus:
   address: ""
+  auth:
+    incluster: false
+    config: ""
+  metrics:
+    relistInterval: 10m
+    maxAge: ""
+    config:
+      resourceRules:
+        cpu:
+          containerQuery: |-
+            sum by (<<.GroupBy>>) (
+              irate(container_cpu_usage_seconds_total{container!="",container!="POD",<<.LabelMatchers>>}[3m])
+            )
+          readyPodsOnlyContainerQuery: |-
+            sum by (<<.GroupBy>>) (
+                (kube_pod_status_ready{condition="true"} == 1)
+              * on (namespace, pod) group_left ()
+                sum by (namespace, pod) (
+                  irate(container_cpu_usage_seconds_total{container!="",container!="POD",<<.LabelMatchers>>}[3m])
+                )
+            )
+          resources:
+            overrides:
+              namespace:
+                resource: namespace
+              pod:
+                resource: pod
+          containerLabel: container
+        memory:
+          containerQuery: |-
+            sum by (<<.GroupBy>>) (
+              container_memory_working_set_bytes{container!="",container!="POD",<<.LabelMatchers>>}
+            )
+          readyPodsOnlyContainerQuery: |-
+            sum by (<<.GroupBy>>) (
+                (kube_pod_status_ready{condition="true"} == 1)
+              * on (namespace, pod) group_left ()
+                sum by (namespace, pod) (
+                  container_memory_working_set_bytes{container!="",container!="POD",<<.LabelMatchers>>}
+                )
+            )
+          resources:
+            overrides:
+              namespace:
+                resource: namespace
+              pod:
+                resource: pod
+          containerLabel: container
+        window: 3m
+      rules: []
+      externalRules:
+      - seriesQuery: '{__name__="kube_pod_status_ready"}'
+        metricsQuery: sum(<<.Series>>{condition="true",<<.LabelMatchers>>})
+        name:
+          as: ready_pods_count
+        resources:
+          overrides:
+            namespace:
+              resource: namespace
+      workloadPodNamePatterns:
+      - group: apps
+        kind: ReplicaSet
+        pattern: ^%s-[a-z0-9]+$
+      - group: apps
+        kind: Deployment
+        pattern: ^%s-[a-z0-9]+-[a-z0-9]+$
+      - group: apps
+        kind: StatefulSet
+        pattern: ^%s-[0-9]+$
 
 webhook:
   enabled: true
   useCertManager: true
 
+algorithmJob:
+  defaultServiceAccount: kapacity-algorithm-job
+  defaultMetricsServerAddr: kapacity-grpc-service:9090
+  defaultImage:
+    horizontalPredictive:
+      registry: ghcr.io
+      repository: traas-stack/kapacity-algorithm-horizontal-predictive
+      tag: v0.1.0
+
 # Set a path to enable logging to disk.
 logPath: ""
 logFileMaxAge: 168h