Skip to content

Commit b8a51de

Browse files
ipeluffoipeluffo
committed
Source code for guide
1 parent 572d7e7 commit b8a51de

14 files changed

+290
-0
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.pyc

‎airflow-envvars-configmap.yaml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: airflow-envvars-configmap
5+
data:
6+
EXECUTOR: Kubernetes
7+
POSTGRES_HOST: postgres
8+
POSTGRES_USER: airflow
9+
POSTGRES_PASSWORD: airflow
10+
POSTGRES_DB: airflow
11+
POSTGRES_PORT: "5432"
12+
LOAD_EX: "y"
13+
# The conf below is necessary because of a typo in the config on docker-airflow image:
14+
# https://github.com/puckel/docker-airflow/blob/bed777970caa3e555ef618d84be07404438c27e3/config/airflow.cfg#L934
15+
AIRFLOW__KUBERNETES__KUBE_CLIENT_REQUEST_ARGS: '{"_request_timeout": [60,60]}'
16+
AIRFLOW__KUBERNETES__WORKER_CONTAINER_REPOSITORY: puckel/docker-airflow
17+
AIRFLOW__KUBERNETES__WORKER_CONTAINER_TAG: "1.10.9"
18+
AIRFLOW__KUBERNETES__DAGS_VOLUME_HOST: /mnt/airflow/dags
19+
AIRFLOW__KUBERNETES__LOGS_VOLUME_CLAIM: airflow-logs-pvc
20+
AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF: airflow-envvars-configmap

‎airflow-rbac.yaml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
kind: ClusterRole
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
metadata:
4+
name: pods-permissions
5+
rules:
6+
- apiGroups: [""]
7+
resources: ["pods"]
8+
verbs: ["get", "list", "watch", "create", "delete"]
9+
10+
---
11+
12+
kind: ClusterRoleBinding
13+
apiVersion: rbac.authorization.k8s.io/v1
14+
metadata:
15+
name: pods-permissions
16+
subjects:
17+
- kind: ServiceAccount
18+
name: default
19+
namespace: default
20+
roleRef:
21+
kind: ClusterRole
22+
name: pods-permissions
23+
apiGroup: rbac.authorization.k8s.io

‎airflow-scheduler-deployment.yaml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: airflow-scheduler
5+
labels:
6+
app: airflow-k8s
7+
8+
spec:
9+
selector:
10+
matchLabels:
11+
app: airflow-scheduler
12+
13+
replicas: 1
14+
15+
template:
16+
metadata:
17+
labels:
18+
app: airflow-scheduler
19+
20+
spec:
21+
containers:
22+
- name: airflow-scheduler
23+
image: puckel/docker-airflow:1.10.9
24+
args: ["scheduler"]
25+
envFrom:
26+
- configMapRef:
27+
name: airflow-envvars-configmap
28+
resources:
29+
limits:
30+
memory: "512Mi"
31+
# cpu: "100"
32+
volumeMounts:
33+
- name: requirements-configmap
34+
subPath: "requirements.txt"
35+
mountPath: "/requirements.txt"
36+
- name: dags-host-volume
37+
mountPath: /usr/local/airflow/dags
38+
- name: logs-persistent-storage
39+
mountPath: /usr/local/airflow/logs
40+
volumes:
41+
- name: requirements-configmap
42+
configMap:
43+
name: requirements-configmap
44+
- name: dags-host-volume
45+
hostPath:
46+
path: /mnt/airflow/dags
47+
type: Directory
48+
- name: logs-persistent-storage
49+
persistentVolumeClaim:
50+
claimName: airflow-logs-pvc

‎airflow-webserver-deployment.yaml

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: airflow-webserver
5+
labels:
6+
app: airflow-k8s
7+
8+
spec:
9+
selector:
10+
matchLabels:
11+
app: airflow-webserver
12+
13+
replicas: 1
14+
15+
template:
16+
metadata:
17+
labels:
18+
app: airflow-webserver
19+
20+
spec:
21+
containers:
22+
- name: airflow-webserver
23+
image: puckel/docker-airflow:1.10.9
24+
envFrom:
25+
- configMapRef:
26+
name: airflow-envvars-configmap
27+
resources:
28+
limits:
29+
memory: "2Gi"
30+
# cpu: "100"
31+
ports:
32+
- containerPort: 8080
33+
volumeMounts:
34+
- name: requirements-configmap
35+
subPath: "requirements.txt"
36+
mountPath: "/requirements.txt"
37+
- name: dags-host-volume
38+
mountPath: /usr/local/airflow/dags
39+
- name: logs-persistent-storage
40+
mountPath: /usr/local/airflow/logs
41+
volumes:
42+
- name: requirements-configmap
43+
configMap:
44+
name: requirements-configmap
45+
- name: dags-host-volume
46+
hostPath:
47+
path: /mnt/airflow/dags
48+
type: Directory
49+
- name: logs-persistent-storage
50+
persistentVolumeClaim:
51+
claimName: airflow-logs-pvc

‎airflow-webserver-service.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: airflow-webserver
5+
labels:
6+
app: airflow-k8s
7+
8+
spec:
9+
type: NodePort
10+
11+
selector:
12+
app: airflow-webserver
13+
14+
ports:
15+
- port: 8080

‎dags/tuto.py

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""
2+
Code that goes along with the Airflow located at:
3+
http://airflow.readthedocs.org/en/latest/tutorial.html
4+
"""
5+
from airflow import DAG
6+
from airflow.operators.bash_operator import BashOperator
7+
from datetime import datetime, timedelta
8+
9+
10+
default_args = {
11+
"owner": "airflow",
12+
"depends_on_past": False,
13+
"start_date": datetime(2015, 6, 1),
14+
"email": ["[email protected]"],
15+
"email_on_failure": False,
16+
"email_on_retry": False,
17+
"retries": 1,
18+
"retry_delay": timedelta(minutes=5),
19+
# 'queue': 'bash_queue',
20+
# 'pool': 'backfill',
21+
# 'priority_weight': 10,
22+
# 'end_date': datetime(2016, 1, 1),
23+
}
24+
25+
dag = DAG("tutorial", default_args=default_args, schedule_interval=timedelta(1))
26+
27+
# t1, t2 and t3 are examples of tasks created by instantiating operators
28+
t1 = BashOperator(task_id="print_date", bash_command="date", dag=dag)
29+
30+
t2 = BashOperator(task_id="sleep", bash_command="sleep 5", retries=3, dag=dag)
31+
32+
templated_command = """
33+
{% for i in range(5) %}
34+
echo "{{ ds }}"
35+
echo "{{ macros.ds_add(ds, 7)}}"
36+
echo "{{ params.my_param }}"
37+
{% endfor %}
38+
"""
39+
40+
t3 = BashOperator(
41+
task_id="templated",
42+
bash_command=templated_command,
43+
params={"my_param": "Parameter I passed in"},
44+
dag=dag,
45+
)
46+
47+
t2.set_upstream(t1)
48+
t3.set_upstream(t1)

‎logs-persistenvolumeclaim.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: v1
2+
kind: PersistentVolumeClaim
3+
metadata:
4+
name: airflow-logs-pvc
5+
labels:
6+
app: airflow-k8s
7+
spec:
8+
accessModes:
9+
- ReadWriteMany
10+
resources:
11+
requests:
12+
storage: 512Mi

‎postgres-deployment.yaml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: postgres
5+
spec:
6+
selector:
7+
matchLabels:
8+
app: postgres
9+
10+
replicas: 1
11+
12+
template:
13+
metadata:
14+
labels:
15+
app: postgres
16+
17+
spec:
18+
containers:
19+
- name: postgres
20+
image: postgres:12
21+
resources:
22+
limits:
23+
memory: 128Mi
24+
cpu: 500m
25+
ports:
26+
- containerPort: 5432
27+
env:
28+
- name: POSTGRES_PASSWORD
29+
value: airflow
30+
- name: POSTGRES_USER
31+
value: airflow
32+
- name: POSTGRES_DB
33+
value: airflow

‎postgres-service.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: postgres
5+
spec:
6+
selector:
7+
app: postgres
8+
9+
ports:
10+
- port: 5432
11+
targetPort: 5432

‎requirements-configmap.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: requirements-configmap
5+
data:
6+
requirements.txt: |
7+
apache-airflow[kubernetes]==1.10.9

‎requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
apache-airflow[kubernetes]==1.10.9

‎script-apply.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
kubectl apply -f logs-persistenvolumeclaim.yaml
2+
kubectl apply -f airflow-rbac.yaml
3+
kubectl apply -f postgres-service.yaml
4+
kubectl apply -f postgres-deployment.yaml
5+
kubectl apply -f requirements-configmap.yaml
6+
kubectl apply -f airflow-envvars-configmap.yaml
7+
kubectl apply -f airflow-webserver-service.yaml
8+
kubectl apply -f airflow-webserver-deployment.yaml
9+
kubectl apply -f airflow-scheduler-deployment.yaml

‎script-delete.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
kubectl delete -f airflow-rbac.yaml
2+
kubectl delete -f postgres-service.yaml
3+
kubectl delete -f postgres-deployment.yaml
4+
kubectl delete -f requirements-configmap.yaml
5+
kubectl delete -f airflow-envvars-configmap.yaml
6+
kubectl delete -f airflow-webserver-service.yaml
7+
kubectl delete -f airflow-webserver-deployment.yaml
8+
kubectl delete -f airflow-scheduler-deployment.yaml
9+
kubectl delete -f logs-persistenvolumeclaim.yaml

0 commit comments

Comments
 (0)