Skip to content

Commit

Permalink
feat: add support for embedded etcd (#213)
Browse files Browse the repository at this point in the history
* feat: add support for embedded etcd

Signed-off-by: Vishal Choudhary <[email protected]>

---------

Signed-off-by: Vishal Choudhary <[email protected]>
  • Loading branch information
vishal-chdhry authored Oct 31, 2024
1 parent c0d75ef commit 5579039
Show file tree
Hide file tree
Showing 19 changed files with 802 additions and 57 deletions.
3 changes: 3 additions & 0 deletions .github/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ nodes:
- containerPort: 443
hostPort: 443
protocol: TCP
extraMounts:
- hostPath: /home/tmp
containerPath: /data
- role: worker
- role: worker
- role: worker
6 changes: 5 additions & 1 deletion .github/workflows/conformance-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,11 @@ jobs:
- name: Install latest kyverno
run: |
set -e
kubectl create -f https://github.com/kyverno/kyverno/raw/main/config/install-latest-testing.yaml
set -e
export HELM=${{ steps.helm.outputs.helm-path }}
helm repo add kyverno https://kyverno.github.io/kyverno/
kubectl create namespace kyverno
helm install kyverno --namespace kyverno kyverno/kyverno --set features.policyExceptions.enabled=true --set features.policyExceptions.namespace='*'
- name: Wait for kyverno ready
run: |
set -e
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/migration-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,13 @@ jobs:
run: |
set -e
kind create cluster --image kindest/node:${{ matrix.k8s-version.version }} --config ./.github/kind.yml
- name: Install kyverno v1.12.4
- name: Install kyverno
run: |
set -e
kubectl create -f https://github.com/kyverno/kyverno/raw/main/config/install-latest-testing.yaml
export HELM=${{ steps.helm.outputs.helm-path }}
helm repo add kyverno https://kyverno.github.io/kyverno/
kubectl create namespace kyverno
helm install kyverno --namespace kyverno kyverno/kyverno --set features.policyExceptions.enabled=true --set features.policyExceptions.namespace='*'
- name: Wait for kyverno ready
run: |
set -e
Expand Down
14 changes: 7 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -164,23 +164,23 @@ codegen-install-manifest: $(HELM) ## Create install manifest
| $(SED) -e '/^#.*/d' \
> ./config/install.yaml

codegen-install-manifest-inmemory: $(HELM) ## Create install manifest without postgres
codegen-install-manifest-etcd: $(HELM) ## Create install manifest without postgres
@echo Generate latest install manifest... >&2
@$(HELM) template reports-server --namespace reports-server ./charts/reports-server/ \
--set apiServicesManagement.installApiServices.enabled=true \
--set image.tag=latest \
--set config.debug=true \
--set config.etcd.enabled=true \
--set postgresql.enabled=false \
--set templating.enabled=true \
| $(SED) -e '/^#.*/d' \
> ./config/install-inmemory.yaml
> ./config/install-etcd.yaml

.PHONY: codegen
codegen: ## Rebuild all generated code and docs
codegen: codegen-helm-docs
codegen: codegen-openapi
codegen: codegen-install-manifest
codegen: codegen-install-manifest-inmemory
codegen: codegen-install-manifest-etcd

.PHONY: verify-codegen
verify-codegen: codegen ## Verify all generated code and docs are up to date
Expand Down Expand Up @@ -220,12 +220,12 @@ kind-install: $(HELM) kind-load ## Build image, load it in kind cluster and depl
--set image.repository=$(PACKAGE) \
--set image.tag=$(GIT_SHA)

.PHONY: kind-install-inmemory
kind-install-inmemory: $(HELM) kind-load ## Build image, load it in kind cluster and deploy helm chart
.PHONY: kind-install-etcd
kind-install-etcd: $(HELM) kind-load ## Build image, load it in kind cluster and deploy helm chart
@echo Install chart... >&2
@$(HELM) upgrade --install reports-server --namespace reports-server --create-namespace --wait ./charts/reports-server \
--set image.registry=$(KO_REGISTRY) \
--set config.debug=true \
--set config.etcd.enabled=true \
--set postgresql.enabled=false \
--set image.repository=$(PACKAGE) \
--set image.tag=$(GIT_SHA)
Expand Down
4 changes: 3 additions & 1 deletion charts/reports-server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,9 @@ helm install reports-server --namespace reports-server --create-namespace report
| affinity | object | `{}` | Affinity |
| service.type | string | `"ClusterIP"` | Service type |
| service.port | int | `443` | Service port |
| config.debug | bool | `false` | Enable debug (to use inmemorydatabase) |
| config.etcd.enabled | bool | `false` | |
| config.etcd.endpoints | string | `nil` | |
| config.etcd.insecure | bool | `true` | |
| config.db.secretName | string | `""` | If set, database connection information will be read from the Secret with this name. Overrides `db.host`, `db.name`, `db.user`, and `db.password`. |
| config.db.host | string | `""` | Database host |
| config.db.hostSecretKeyName | string | `"host"` | The database host will be read from this `key` in the specified Secret, when `db.secretName` is set. |
Expand Down
14 changes: 9 additions & 5 deletions charts/reports-server/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,12 @@ spec:
containers:
- name: reports-server
args:
{{- if .Values.config.debug }}
- --debug
{{- if .Values.config.etcd.enabled }}
- --etcd
{{- if .Values.config.etcd.insecure }}
- --etcdSkipTLS
{{- end }}
- --etcdEndpoints=https://etcd-0.etcd.{{ $.Release.Namespace }}:2379,https://etcd-1.etcd.{{ $.Release.Namespace }}:2379,https://etcd-2.etcd.{{ $.Release.Namespace }}:2379
{{- else }}
- --dbhost={{ include "reports-server.dbHost" . }}
- --dbport={{ include "reports-server.dbPort" . }}
Expand Down Expand Up @@ -85,15 +89,15 @@ spec:
{{- end}}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
volumeMounts:
- mountPath: /tmp
name: tmp-dir
image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: https
containerPort: 4443
protocol: TCP
volumeMounts:
- mountPath: /tmp
name: tmp-dir
{{- with .Values.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
Expand Down
170 changes: 170 additions & 0 deletions charts/reports-server/templates/etcd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
{{- if .Values.config.etcd.enabled }}
---
apiVersion: v1
kind: Service
metadata:
name: etcd
namespace: {{ $.Release.Namespace }}
labels:
app: etcd-reports-server
{{- include "reports-server.labels" . | nindent 4 }}
spec:
type: ClusterIP
clusterIP: None
selector:
app: etcd-reports-server
publishNotReadyAddresses: true
ports:
- name: etcd-client
port: 2379
- name: etcd-server
port: 2380
- name: etcd-metrics
port: 8080
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: {{ include "reports-server.fullname" . }}
name: etcd
labels:
app: etcd-reports-server
{{- include "reports-server.labels" . | nindent 4 }}
spec:
serviceName: etcd
replicas: 3
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
app: etcd-reports-server
template:
metadata:
labels:
app: etcd-reports-server
annotations:
serviceName: etcd
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- etcd-reports-server
topologyKey: "kubernetes.io/hostname"
containers:
- name: etcd
image: quay.io/coreos/etcd:v3.5.15
imagePullPolicy: IfNotPresent
ports:
- name: etcd-client
containerPort: 2379
- name: etcd-server
containerPort: 2380
- name: etcd-metrics
containerPort: 8080
readinessProbe:
httpGet:
path: /readyz
port: 8080
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: /livez
port: 8080
initialDelaySeconds: 15
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3
env:
- name: K8S_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: HOSTNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: SERVICE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.annotations['serviceName']
- name: ETCDCTL_ENDPOINTS
value: $(HOSTNAME).$(SERVICE_NAME):2379
## TLS client configuration for etcdctl in the container.
## These files paths are part of the "etcd-client-certs" volume mount.
# - name: ETCDCTL_KEY
# value: /etc/etcd/certs/client/tls.key
# - name: ETCDCTL_CERT
# value: /etc/etcd/certs/client/tls.crt
# - name: ETCDCTL_CACERT
# value: /etc/etcd/certs/client/ca.crt
##
## Use this URI_SCHEME value for non-TLS clusters.
- name: URI_SCHEME
value: "http"
## TLS: Use this URI_SCHEME for TLS clusters.
# - name: URI_SCHEME
# value: "https"
command:
- /usr/local/bin/etcd
args:
- --name=$(HOSTNAME)
- --data-dir=/data
- --wal-dir=/data/wal
- --listen-peer-urls=$(URI_SCHEME)://0.0.0.0:2380
- --listen-client-urls=$(URI_SCHEME)://0.0.0.0:2379
- --advertise-client-urls=$(URI_SCHEME)://$(HOSTNAME).$(SERVICE_NAME):2379
- --initial-cluster-state=new
- --initial-cluster-token=etcd-$(K8S_NAMESPACE)
- --initial-cluster=etcd-0=$(URI_SCHEME)://etcd-0.$(SERVICE_NAME):2380,etcd-1=$(URI_SCHEME)://etcd-1.$(SERVICE_NAME):2380,etcd-2=$(URI_SCHEME)://etcd-2.$(SERVICE_NAME):2380
- --initial-advertise-peer-urls=$(URI_SCHEME)://$(HOSTNAME).$(SERVICE_NAME):2380
- --listen-metrics-urls=http://0.0.0.0:8080
# - --auto-compaction-mode=periodic
# - --auto-compaction-retention=10m
# - --client-cert-auth
# - --trusted-ca-file=$(ETCDCTL_CACERT)
# - --cert-file=$(ETCDCTL_CERT)
# - --key-file=$(ETCDCTL_KEY)
# - --peer-client-cert-auth
# - --peer-trusted-ca-file=/etc/etcd/certs/server/ca.crt
# - --peer-cert-file=/etc/etcd/certs/server/tls.crt
# - --peer-key-file=/etc/etcd/certs/server/tls.key
volumeMounts:
- name: etcd-data
mountPath: /data
# - name: etcd-client-tls
# mountPath: "/etc/etcd/certs/client"
# readOnly: true
# - name: etcd-server-tls
# mountPath: "/etc/etcd/certs/server"
# readOnly: true
volumes:
# - name: etcd-client-tls
# secret:
# secretName: etcd-client-tls
# optional: false
# - name: etcd-server-tls
# secret:
# secretName: etcd-server-tls
# optional: false
volumeClaimTemplates:
- metadata:
name: etcd-data
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 1Gi
{{- end }}

6 changes: 4 additions & 2 deletions charts/reports-server/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,10 @@ service:

config:

# -- Enable debug (to use inmemorydatabase)
debug: false
etcd:
enabled: false
endpoints: ~
insecure: true

db:
# -- If set, database connection information will be read from the Secret with this name. Overrides `db.host`, `db.name`, `db.user`, and `db.password`.
Expand Down
Loading

0 comments on commit 5579039

Please sign in to comment.