Skip to content

Commit 8fc53d6

Browse files
ShubhamTiwary914ShubhamTiwary914
authored
Merge pull request #28 from ShubhamTiwary914/prod/provision
beametl k8s deployment & helm charts with GHA config
2 parents 25ec346 + 7e90eea commit 8fc53d6

File tree

14 files changed

+193
-20
lines changed

14 files changed

+193
-20
lines changed

‎.github/workflows/helm-beam-chart-release.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Helm Chart release for Beametl to charts branch
2+
3+
on:
4+
push:
5+
branches:
6+
- 'master'
7+
paths:
8+
- 'helm/beametl/**'
9+
workflow_dispatch:
10+
11+
jobs:
12+
release-helm-verne:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout repo
16+
uses: actions/checkout@v5
17+
with:
18+
token: ${{ secrets.REPO_ACCESS_TOKEN }}
19+
fetch-depth: 0
20+
21+
- name: Set up Helm
22+
uses: azure/[email protected]
23+
with:
24+
version: 'latest'
25+
26+
- name: Package Helm chart to tempdir
27+
run: helm package helm/beametl --destination /tmp/helm-charts
28+
29+
- name: Switch to charts branch
30+
run: |
31+
git fetch origin charts
32+
git checkout charts
33+
34+
- name: Copy packaged chart into beametl folder
35+
run: |
36+
mkdir -p ./beametl
37+
cp /tmp/helm-charts/*.tgz ./beametl
38+
39+
- name: Generate Helm repo index
40+
run: |
41+
helm repo index ./beametl --url https://raw.githubusercontent.com/${{ github.repository }}/charts/beametl
42+
43+
- name: Commit and push
44+
run: |
45+
git config user.name "github-actions[bot]"
46+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
47+
git add ./beametl
48+
git commit -m "Release beametl chart from ${{ github.sha }}" || echo "No changes"
49+
git push origin charts

‎.gitignore‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,8 @@
44
**/__pycache__
55
**/node_modules
66
**/graf-data
7-
**/*.log
7+
**/*.log
8+
9+
terraform/**.tfstate*
10+
terraform/.terraform
11+
terraform/**.lock*

‎helm/beametl/Chart.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: v2
2+
name: beametl
3+
description: BeamETL pipeline Kubernetes deployment helm chart
4+
5+
type: application
6+
version: 0.1.0
7+
8+
appVersion: "1.16.0"

‎helm/beametl/templates/deployment.yaml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# yaml-language-server: $schema=https://json.schemastore.org/helmfile
2+
# beam streaming pipeline with DirectRunner to connect pubsub <-> bigtable
3+
# flow context:
4+
# - source: pubsub(topic: mqtt-source)
5+
# - transformation: col families: payload,relation, convert to cbt object
6+
# - sink: cbt(relation: iotsink)
7+
---
8+
apiVersion: apps/v1
9+
kind: Deployment
10+
metadata:
11+
name: beametl-pod
12+
namespace: {{ .Values.namespace }}
13+
labels:
14+
app: beampod
15+
spec:
16+
replicas: {{ .Values.replicaCount }}
17+
selector:
18+
matchLabels:
19+
app: beampod
20+
template:
21+
metadata:
22+
labels:
23+
app: beampod
24+
spec:
25+
containers:
26+
#assumes GCP ADC is provided on the GKE cluster's pods for pubsub/bigtable access
27+
- name: beametl-container
28+
image: {{ .Values.image }}
29+
resources:
30+
{{- with .Values.resourceOptions }}
31+
requests:
32+
{{- with .requests }}
33+
memory: {{ .memory | quote }}
34+
cpu: {{ .cpu | quote }}
35+
{{- end }}
36+
limits:
37+
{{- with .limits }}
38+
memory: {{ .memory | quote }}
39+
cpu: {{ .cpu | quote }}
40+
{{- end }}
41+
{{- end }}
42+
---

‎helm/beametl/values.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
namespace: beametl
2+
replicaCount: 1
3+
image: sardinesszsz/beametl-pubsub-to-cbt:latest
4+
5+
resourceOptions:
6+
requests:
7+
memory: "512Mi"
8+
cpu: "2"
9+
limits:
10+
memory: "1.5Gi"
11+
cpu: "4"

‎helm/verne/.helmignore‎

Whitespace-only changes.

‎helm/verne/templates/deployment.yaml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# yaml-language-server: $schema=https://json.schemastore.org/helmfile
2-
#vernMQTT pods deployment config with sidecar for listener
3-
#listener -> a golang script (just intake message bytes) -> push to output stream (in this case, pub/sub)
4-
2+
# vernMQTT pods deployment config with sidecar for listener
3+
# listener -> golang script (just intake message bytes) -> push to output stream (in this case, pub/sub)
4+
---
55
apiVersion: apps/v1
66
kind: Deployment
77
metadata:
@@ -117,4 +117,5 @@ spec:
117117
periodSeconds: 2
118118
volumeMounts:
119119
- name: envs
120-
mountPath: /envs
120+
mountPath: /envs
121+
---

‎helm/verne/values.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
replicaCount: 2
22
env: dev
33
namespace: verne
4-
image: sardinesszsz/verne-listener:latest
4+
image: sardinesszsz/verne-listener:latest
55

66
verneConfig:
77
acceptEula: "yes"

‎k8s/prod/beametl/values.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
namespace: beametl
2+
replicaCount: 1
3+
image: sardinesszsz/beametl-pubsub-to-cbt:latest
4+
5+
resourceOptions:
6+
requests:
7+
memory: "512Mi"
8+
cpu: "2"
9+
limits:
10+
memory: "1.5Gi"
11+
cpu: "4"

‎scripts/run-local.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ docker pull sardinesszsz/verne-listener:latest
3535

3636
# Add Helm repositories
3737
log_info "Adding Helm repositories..."
38-
helm repo add verne https://raw.githubusercontent.com/ShubhamTiwary914/logcore-v2/charts/ 2>/dev/null || log_skip "Verne repo already added"
38+
helm repo add verne https://raw.githubusercontent.com/ShubhamTiwary914/logcore-v2/charts/verne 2>/dev/null || log_skip "Verne repo already added"
3939
helm repo add grafana https://grafana.github.io/helm-charts 2>/dev/null || log_skip "Grafana repo already added"
4040
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 2>/dev/null || log_skip "Prometheus repo already added"
4141
helm repo update

0 commit comments

Comments
 (0)