diff --git a/examples/README.md b/examples/README.md index 8d7c9dd..ead5b47 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,3 +1,54 @@ -# carbon cronjob +# examples -This cronjob queries an existing Prometheus for `entsoe_generation_co2` and makes decission how many resourcequotas are allowed for the workload in the running namespace. +Some show cased howto operate workload based on carbon emission. + +Precondition: A installed [Prometheus Stack with Entsoe Exporter](https://github.com/eumel8/carbon-footprint) to provide the current carbon footprint of power generation. + +## Update resourcequotas per Cron + +In [this cronjob](quota/carbon-cronjob.yaml) a Prometheus API will ask for current carbon state of power generation. +On this decision `resourcequotas` for the target namespace will adjusted. + +Works technically but has no effects on running workload or the workload won't start if the quota is reached + +## Modify cgroups/cpu.max in Pod + +Resourcequotas are realized by cgroup settings in Kubelet and the underlying [Cgroup Driver](https://kubernetes.io/docs/concepts/architecture/cgroups/), which manifests by the underlying Container Runtime Interface(CRI). Usually, and without HostPath this resources are only read-only in the Pod and can't be modified. [Alibaba Cloud](https://www.alibabacloud.com/help/en/ack/ack-managed-and-ack-dedicated/user-guide/dynamically-modify-the-resource-parameters-of-a-pod) has a cgroup controller running, so the user can do this and act as in the example above + +## In-Place update Pod resources + +In Kubernetes 1.27 this [Kep](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/kep.yaml) was realized, which makes the resouces in containers.spec writable. + +Requires, like K3S start flag: + +```bash +... + --kube-apiserver-arg feature-gates="InPlacePodVerticalScaling=true" \ + --kube-controller-arg feature-gates="InPlacePodVerticalScaling=true" \ + --kubelet-arg feature-gates="InPlacePodVerticalScaling=true" \ +``` + +Then you can use the carbon-cronjob.yaml and make a patch based on the current carbon emission: + +```bash +kubectl -n carbon patch pod pod-demo --patch '{"spec":{"containers":[{"name":"pod-demo", "resources":{"requests":{"cpu":"550m"}}}]}}' +pod/pod-demo patched +``` + +## Deployment Resources + +see [./resources/](resources) + +Patch deployment and adjust cpu resources based on eco power generation + +## Horizontal Pod Autoscaler (HPA) + +see [./hpa/](hpa) + +Patch hpa and adjust replicas based on eco power generation + +## Keda Prometheus + +see [./keda-prometheus/](keda-prometheus) + +Use ScaledObject from [https://keda.io](Keda) to fetch Prometheus metrics of carbon emission and act on workload deployment diff --git a/examples/hpa/carbon-cronjob.yaml b/examples/hpa/carbon-cronjob.yaml new file mode 100644 index 0000000..e4a9f29 --- /dev/null +++ b/examples/hpa/carbon-cronjob.yaml @@ -0,0 +1,130 @@ +# This cronjob queries an existing Prometheus for `entsoe_generation_co2` +# and makes decission how many resourcequotas are allowed for the workload in the running namespace. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +rules: +- apiGroups: + - "autoscaling" + resources: + - horizontalpodautoscalers + verbs: + - create + - get + - list + - watch + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: carbon-cronjob +subjects: + - kind: ServiceAccount + name: carbon-cronjob +--- +apiVersion: v1 +data: + run.sh: | + #!/bin/bash + # set 3 carbon emission limits to operate resource quotas + highlimit=80 + middlelimit=50 + smalllimit=40 + quota=10 + # get the current carbon emission from entsoe + carbon=$(curl -s http://carbon-prometheus-server:80/api/v1/query?query=entsoe_generation_eco | jq -r '.data.result[]|select(.metric.job=="entsoe-carbon-footprint")|.value[-1]') + if [ "${#carbon}" -lt 1 ]; then + echo "no carbon emission data at this time" + exit + fi + # unit is g/s, multiplicate and round because bash can't handle floating numbers + carbon=$(printf "%.0f\n" $( bc -l <<<"100*$carbon" )) + echo "carbon factor is "$carbon" , operate the deployment now" + # set the cpu resource limit based on the current carbon emission factor + if [[ $carbon -gt $highlimit ]]; then + kubectl patch hpa demoapp --patch '{"spec":{"maxReplicas":4}}' + elif [[ $carbon -gt $middlelimit ]] && [[ $carbon -lt $highlimit ]]; then + kubectl patch hpa demoapp --patch '{"spec":{"maxReplicas":3}}' + elif [[ $carbon -lt $middlelimit ]] && [[ $carbon -gt $smalllimit ]]; then + kubectl patch hpa demoapp --patch '{"spec":{"maxReplicas":2}}' + elif [[ $carbon -lt $smalllimit ]] && [[ $carbon -gt 0 ]]; then + kubectl patch hpa demoapp --patch '{"spec":{"maxReplicas":1}}' + fi + +kind: ConfigMap +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + job-name: carbon-cronjob + name: carbon-cronjob +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + suspend: false + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - image: mtr.devops.telekom.de/caas/k8s-tools:latest + imagePullPolicy: Always + name: carbon-cronjob + command: ["sh","-c"] + args: ["/sidecar/run.sh"] + resources: + limits: + cpu: 400m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - name: carbon-cronjob + mountPath: /sidecar + securityContext: + fsGroup: 1000 + supplementalGroups: + - 1000 + serviceAccountName: carbon-cronjob + volumes: + - name: carbon-cronjob + configMap: + defaultMode: 0755 + name: carbon-cronjob diff --git a/examples/hpa/deployment.yaml b/examples/hpa/deployment.yaml new file mode 100644 index 0000000..2596def --- /dev/null +++ b/examples/hpa/deployment.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: demoapp +spec: + selector: + matchLabels: + app: demoapp + replicas: 1 + template: + metadata: + labels: + app: demoapp + spec: + containers: + - args: + - --vm + - "1" + - --vm-bytes + - 256M + - -c + - "2" + - --vm-hang + - "1" + command: + - stress + name: demoapp + image: mtr.devops.telekom.de/caas/stress:latest + resources: + requests: + cpu: 1000m + memory: 256Mi + limits: + cpu: 1000m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + stdin: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + restartPolicy: Always + securityContext: + fsGroup: 1000 + supplementalGroups: + - 1000 + terminationGracePeriodSeconds: 1 diff --git a/examples/hpa/hpa.yaml b/examples/hpa/hpa.yaml new file mode 100644 index 0000000..d0ab62e --- /dev/null +++ b/examples/hpa/hpa.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: demoapp +spec: + maxReplicas: 1 + metrics: + - resource: + name: cpu + target: + averageUtilization: 90 + type: Utilization + type: Resource + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: demoapp diff --git a/examples/keda-prometheus/prometheus-scale.yaml b/examples/keda-prometheus/prometheus-scale.yaml new file mode 100644 index 0000000..e64db5a --- /dev/null +++ b/examples/keda-prometheus/prometheus-scale.yaml @@ -0,0 +1,34 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: carbon-keda-scaledobject +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: demoapp + pollingInterval: 10 # Optional. Default: 30 seconds + cooldownPeriod: 300 # Optional. Default: 300 seconds + minReplicaCount: 1 # Optional. Default: 0 + maxReplicaCount: 4 # Optional. Default: 100 + fallback: # Optional. Section to specify fallback options + failureThreshold: 3 # Mandatory if fallback section is included + replicas: 1 + advanced: # Optional. Section to specify advanced options + horizontalPodAutoscalerConfig: # Optional. Section to specify HPA related options + behavior: # Optional. Use to modify HPA's scaling behavior + scaleDown: + stabilizationWindowSeconds: 150 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + triggers: + - type: prometheus + metadata: + serverAddress: http://carbon-prometheus-server.carbon + metricName: entsoe_generation_eco + query: entsoe_generation_eco{job="entsoe-carbon-footprint"}*100 + threshold: '65' + + diff --git a/examples/carbon-cronjob.yaml b/examples/quota/carbon-cronjob.yaml similarity index 95% rename from examples/carbon-cronjob.yaml rename to examples/quota/carbon-cronjob.yaml index 020a942..7edb4e8 100644 --- a/examples/carbon-cronjob.yaml +++ b/examples/quota/carbon-cronjob.yaml @@ -1,3 +1,5 @@ +# This cronjob queries an existing Prometheus for `entsoe_generation_co2` +# and makes decission how many resourcequotas are allowed for the workload in the running namespace. --- apiVersion: v1 kind: ServiceAccount diff --git a/examples/resources/carbon-cronjob.yaml b/examples/resources/carbon-cronjob.yaml new file mode 100644 index 0000000..405beb4 --- /dev/null +++ b/examples/resources/carbon-cronjob.yaml @@ -0,0 +1,130 @@ +# This cronjob queries an existing Prometheus for `entsoe_generation_co2` +# and makes decission how many resourcequotas are allowed for the workload in the running namespace. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +rules: +- apiGroups: + - "apps" + resources: + - deployments + verbs: + - create + - get + - list + - watch + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: carbon-cronjob +subjects: + - kind: ServiceAccount + name: carbon-cronjob +--- +apiVersion: v1 +data: + run.sh: | + #!/bin/bash + # set 3 carbon emission limits to operate resource quotas + highlimit=80 + middlelimit=50 + smalllimit=40 + quota=10 + # get the current carbon emission from entsoe + carbon=$(curl -s http://carbon-prometheus-server:80/api/v1/query?query=entsoe_generation_eco | jq -r '.data.result[]|select(.metric.job=="entsoe-carbon-footprint")|.value[-1]') + if [ "${#carbon}" -lt 1 ]; then + echo "no carbon emission data at this time" + exit + fi + # unit is g/s, multiplicate and round because bash can't handle floating numbers + carbon=$(printf "%.0f\n" $( bc -l <<<"100*$carbon" )) + echo "carbon factor is "$carbon" , operate the deployment now" + # set the cpu resource limit based on the current carbon emission factor + if [[ $carbon -gt $highlimit ]]; then + kubectl patch deployment demoapp --patch '{"spec":{"template":{"spec":{"containers":[{"name":"demoapp", "resources":{"limits":{"cpu":"1000m"},"requests":{"cpu":"1000m"}}}]}}}}' + elif [[ $carbon -gt $middlelimit ]] && [[ $carbon -lt $highlimit ]]; then + kubectl patch deployment demoapp --patch '{"spec":{"template":{"spec":{"containers":[{"name":"demoapp", "resources":{"limits":{"cpu":"800m"},"requests":{"cpu":"800m"}}}]}}}}' + elif [[ $carbon -lt $middlelimit ]] && [[ $carbon -gt $smalllimit ]]; then + kubectl patch deployment demoapp --patch '{"spec":{"template":{"spec":{"containers":[{"name":"demoapp", "resources":{"limits":{"cpu":"500m"},"requests":{"cpu":"500m"}}}]}}}}' + elif [[ $carbon -lt $smalllimit ]] && [[ $carbon -gt 0 ]]; then + kubectl patch deployment demoapp --patch '{"spec":{"template":{"spec":{"containers":[{"name":"demoapp", "resources":{"limits":{"cpu":"100m"},"requests":{"cpu":"100m"}}}]}}}}' + fi + +kind: ConfigMap +metadata: + labels: + app: carbon-cronjob + name: carbon-cronjob +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + labels: + job-name: carbon-cronjob + name: carbon-cronjob +spec: + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + suspend: false + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - image: mtr.devops.telekom.de/caas/k8s-tools:latest + imagePullPolicy: Always + name: carbon-cronjob + command: ["sh","-c"] + args: ["/sidecar/run.sh"] + resources: + limits: + cpu: 400m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumeMounts: + - name: carbon-cronjob + mountPath: /sidecar + securityContext: + fsGroup: 1000 + supplementalGroups: + - 1000 + serviceAccountName: carbon-cronjob + volumes: + - name: carbon-cronjob + configMap: + defaultMode: 0755 + name: carbon-cronjob diff --git a/examples/resources/deployment.yaml b/examples/resources/deployment.yaml new file mode 100644 index 0000000..2596def --- /dev/null +++ b/examples/resources/deployment.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: demoapp +spec: + selector: + matchLabels: + app: demoapp + replicas: 1 + template: + metadata: + labels: + app: demoapp + spec: + containers: + - args: + - --vm + - "1" + - --vm-bytes + - 256M + - -c + - "2" + - --vm-hang + - "1" + command: + - stress + name: demoapp + image: mtr.devops.telekom.de/caas/stress:latest + resources: + requests: + cpu: 1000m + memory: 256Mi + limits: + cpu: 1000m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + stdin: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + restartPolicy: Always + securityContext: + fsGroup: 1000 + supplementalGroups: + - 1000 + terminationGracePeriodSeconds: 1