Skip to content

Commit 45a0afc

Browse files
kahkhangkahkhang
authored
Updates to rook / kube-dashboard / traefik (#78)
Rook environmental variable FLEXVOLUME_DIR_PATH is now set Traefik is now a daemonset Kube-Dashboard is bumped to v1.8.1 A dedicated block device is now created (recommended for Ceph Bluestore)
1 parent 5470315 commit 45a0afc

7 files changed

+63
-61
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ resolv.conf
1111
bootkube
1212
manifests/grafana/grafana-credentials.yaml
1313
install.exp
14+
manifests/container-linux/master-config.yaml.bak

‎install-coreos.sh

+2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/bin/bash
22
set -euo pipefail
3+
[[ -n "$REBOOT_STRATEGY" ]] || die "Need a reboot strategy. Run with eg. '\$REBOOT_STRATEGY=off ./install-coreos.sh'"
4+
35
PUBLIC_IP=$(ip addr show eth0 | grep "inet\b" | grep "/24" | awk '{print $2}' | cut -d/ -f1)
46
PRIVATE_IP=$(ip addr show eth0 | grep "inet\b" | grep "/17" | awk '{print $2}' | cut -d/ -f1)
57

‎linode-utilities.sh

+42-51
Original file line numberDiff line numberDiff line change
@@ -182,22 +182,18 @@ boot_linode() {
182182

183183
update_coreos_config() {
184184
linode_api linode.config.update LinodeID=$LINODE_ID ConfigID=$CONFIG_ID Label="CoreOS" \
185-
DiskList=$DISK_ID KernelID=213 RootDeviceNum=1 helper_network=false
185+
DiskList=$DISK_ID,$STORAGE_DISK_ID KernelID=213 RootDeviceNum=1 helper_network=false
186186
}
187187

188188
transfer_acme() {
189+
IP=$1
189190
ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -tt "core@$IP" \
190191
"sudo truncate -s 0 /etc/traefik/acme/acme.json; echo '$( base64 $base64_args < acme.json )' \
191192
| base64 --decode | sudo tee --append /etc/traefik/acme/acme.json" 2>/dev/null >/dev/null
192193
ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -tt "core@$IP" \
193194
"sudo chmod 600 /etc/traefik/acme/acme.json" 2>/dev/null >/dev/null
194195
}
195196

196-
delete_bootstrap_script() {
197-
ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -tt "core@$IP" \
198-
"rm bootstrap.sh" 2>/dev/null
199-
}
200-
201197
change_to_provisioned() {
202198
local LINODE_ID=$1
203199
local NODE_TYPE=$2
@@ -232,7 +228,6 @@ install() {
232228
local LINODE_ID
233229
local PLAN
234230
local ROOT_PASSWORD
235-
local COREOS_OLD_DISK_SIZE
236231
NODE_TYPE=$1
237232
LINODE_ID=$2
238233
PUBLIC_IP=$(get_public_ip $LINODE_ID)
@@ -241,28 +236,31 @@ install() {
241236
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Retrieving current plan" "get_plan_id $LINODE_ID" PLAN
242237
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Retrieving maximum available disk size" "get_max_disk_size $PLAN" TOTAL_DISK_SIZE
243238

244-
INSTALL_DISK_SIZE=4096
245-
COREOS_OLD_DISK_SIZE=$((${TOTAL_DISK_SIZE}-${INSTALL_DISK_SIZE}))
239+
INSTALL_DISK_SIZE=1024
240+
COREOS_DISK_SIZE=10240
241+
STORAGE_DISK_SIZE=$((${TOTAL_DISK_SIZE}-${COREOS_DISK_SIZE}))
246242

247-
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Creating CoreOS disk" "create_raw_disk $LINODE_ID $COREOS_OLD_DISK_SIZE CoreOS" DISK_ID
243+
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Creating ${COREOS_DISK_SIZE}mb CoreOS disk" "create_raw_disk $LINODE_ID $COREOS_DISK_SIZE CoreOS" DISK_ID
248244

249245
# Create the install OS disk from script
250-
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Creating install disk" create_install_disk INSTALL_DISK_ID
246+
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Creating ${INSTALL_DISK_SIZE}mb install disk" create_install_disk INSTALL_DISK_ID
251247

252248
# Configure the installer to boot
253249
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Creating boot configuration" create_boot_configuration CONFIG_ID
254250
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Booting installer" "boot_linode $LINODE_ID $CONFIG_ID"
255-
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Updating CoreOS config" update_coreos_config
256251
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Installing CoreOS (might take a while)" "install_coreos $LINODE_ID $NODE_TYPE"
257252
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Shutting down CoreOS" "linode_api linode.shutdown LinodeID=$LINODE_ID"
258253
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Deleting install disk $INSTALL_DISK_ID" "linode_api linode.disk.delete LinodeID=$LINODE_ID DiskID=$INSTALL_DISK_ID"
259-
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Resizing CoreOS disk $DISK_ID" "linode_api linode.disk.resize LinodeID=$LINODE_ID DiskID=$DISK_ID Size=$TOTAL_DISK_SIZE"
254+
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Waiting for existing jobs to complete" "wait_jobs $LINODE_ID"
255+
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Creating ${STORAGE_DISK_SIZE}mb storage disk" "create_raw_disk $LINODE_ID $STORAGE_DISK_SIZE Storage" STORAGE_DISK_ID
256+
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Updating CoreOS config" update_coreos_config
257+
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Waiting for existing jobs to complete" "wait_jobs $LINODE_ID"
260258
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Booting CoreOS" "linode_api linode.boot LinodeID=$LINODE_ID ConfigID=$CONFIG_ID"
261259
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Waiting for CoreOS to be ready" "wait_jobs $LINODE_ID; sleep 20"
262260

263261
if [ "$NODE_TYPE" = "master" ] ; then
264262
if [ -e acme.json ] ; then
265-
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Transferring acme.json" transfer_acme
263+
spinner "${CYAN}[$PUBLIC_IP]${NORMAL} Transferring acme.json" "transfer_acme $PUBLIC_IP"
266264
fi
267265
fi
268266

@@ -288,23 +286,24 @@ provision_master() {
288286
[ -e ~/.kube/config ] && mv ~/.kube/config ~/.kube/config.bak
289287
cp cluster/auth/kubeconfig ~/.kube/config
290288
while true; do kubectl --namespace=kube-system create secret generic kubesecret --from-file auth --request-timeout 0 && break || sleep 5; done
291-
if kubectl --request-timeout 0 get namespaces | grep -q "monitoring"; then
292-
echo "namespace monitoring exists"
293-
else
294-
while true; do kubectl create namespace "monitoring" --request-timeout 0 && break || sleep 5; done
295-
fi
296-
if kubectl --request-timeout 0 get namespaces | grep -q "rook"; then
297-
echo "namespace rook exists"
298-
else
299-
while true; do kubectl create namespace "rook" --request-timeout 0 && break || sleep 5; done
300-
fi
289+
cat <<EOF | kubectl apply --request-timeout 0 -f -
290+
apiVersion: v1
291+
kind: Namespace
292+
metadata:
293+
name: monitoring
294+
---
295+
apiVersion: v1
296+
kind: Namespace
297+
metadata:
298+
name: rook
299+
EOF
301300
while true; do kubectl --namespace=monitoring create secret generic kubesecret --from-file auth --request-timeout 0 && break || sleep 5; done
302-
while true; do kubectl apply -f manifests/heapster.yaml --validate=false --request-timeout 0 && break || sleep 5; done
301+
while true; do kubectl apply -f manifests/heapster.yaml --request-timeout 0 && break || sleep 5; done
303302
if [ $INSTALL_K8S_DASHBOARD = true ]; then
304303
while true; do cat manifests/kube-dashboard.yaml | sed "s/\${DOMAIN}/${DOMAIN}/g" | kubectl apply --request-timeout 0 --validate=false -f - && break || sleep 5; done
305304
fi
306305
if [ $INSTALL_TRAEFIK = true ]; then
307-
while true; do cat manifests/traefik.yaml | sed "s/\${DOMAIN}/${DOMAIN}/g" | sed "s/\${MASTER_IP}/${IP}/g" | sed "s/\$EMAIL/${EMAIL}/g" | kubectl apply --request-timeout 0 --validate=false -f - && break || sleep 5; done
306+
while true; do cat manifests/traefik.yaml | sed "s/\${DOMAIN}/${DOMAIN}/g" | sed "s/\$EMAIL/${EMAIL}/g" | kubectl apply --request-timeout 0 --validate=false -f - && break || sleep 5; done
308307
fi
309308
echo "provisioned master"
310309
}
@@ -321,37 +320,29 @@ provision_worker() {
321320

322321
if [ $INSTALL_ROOK = true ]; then
323322
if ! kubectl --namespace rook get pods --request-timeout 0 2>/dev/null | grep -q "^rook-api"; then
324-
if ! kubectl --namespace rook get pods --request-timeout 0 2>/dev/null | grep -q "^rook-api"; then
325-
if ! kubectl --namespace rook get pods --request-timeout 0 2>/dev/null | grep -q "^rook-api"; then
326-
while true; do kubectl apply -f manifests/rook/rook-operator.yaml --request-timeout 0 && break || sleep 5; done
327-
while true; do kubectl apply -f manifests/rook/rook-cluster.yaml --request-timeout 0 && break || sleep 5; done
328-
while true; do kubectl apply -f manifests/rook/rook-storageclass.yaml --request-timeout 0 && break || sleep 5; done
329-
fi
330-
fi
323+
while true; do kubectl apply -f manifests/rook/rook-operator.yaml --request-timeout 0 && break || sleep 5; done
324+
while true; do kubectl apply -f manifests/rook/rook-cluster.yaml --request-timeout 0 && break || sleep 5; done
325+
while true; do kubectl apply -f manifests/rook/rook-storageclass.yaml --request-timeout 0 && break || sleep 5; done
331326
fi
332327
fi
333328

334329
if [ $INSTALL_PROMETHEUS = true ]; then
335330
until kubectl get nodes > /dev/null 2>&1; do sleep 1; done
336331
if ! kubectl --namespace monitoring get ingress --request-timeout 0 2>/dev/null | grep -q "^prometheus-ingress"; then
337-
if ! kubectl --namespace monitoring get ingress --request-timeout 0 2>/dev/null | grep -q "^prometheus-ingress"; then
338-
if ! kubectl --namespace monitoring get ingress --request-timeout 0 2>/dev/null | grep -q "^prometheus-ingress"; then
339-
while true; do kubectl --namespace monitoring apply -f manifests/prometheus-operator --request-timeout 0 && break || sleep 5; done
340-
printf "Waiting for Operator to register third party objects..."
341-
until kubectl --namespace monitoring get servicemonitor > /dev/null 2>&1; do sleep 1; printf "."; done
342-
until kubectl --namespace monitoring get prometheus > /dev/null 2>&1; do sleep 1; printf "."; done
343-
until kubectl --namespace monitoring get alertmanager > /dev/null 2>&1; do sleep 1; printf "."; done
344-
while true; do kubectl --namespace monitoring apply -f manifests/node-exporter --request-timeout 0 && break || sleep 5; done
345-
while true; do kubectl --namespace monitoring apply -f manifests/kube-state-metrics --request-timeout 0 && break || sleep 5; done
346-
while true; do kubectl --namespace monitoring apply -f manifests/grafana/grafana-credentials.yaml --request-timeout 0 && break || sleep 5; done
347-
while true; do kubectl --namespace monitoring apply -f manifests/grafana --request-timeout 0 && break || sleep 5; done
348-
while true; do find manifests/prometheus -type f ! -name prometheus-k8s-roles.yaml ! -name prometheus-k8s-role-bindings.yaml ! -name prometheus-k8s-ingress.yaml -exec kubectl --request-timeout 0 --namespace "monitoring" apply -f {} \; && break || sleep 5; done
349-
while true; do kubectl apply -f manifests/prometheus/prometheus-k8s-roles.yaml --request-timeout 0 && break || sleep 5; done
350-
while true; do kubectl apply -f manifests/prometheus/prometheus-k8s-role-bindings.yaml --request-timeout 0 && break || sleep 5; done
351-
while true; do kubectl --namespace monitoring apply -f manifests/alertmanager/ --request-timeout 0 && break || sleep 5; done
352-
while true; do cat manifests/prometheus/prometheus-k8s-ingress.yaml | sed "s/\${DOMAIN}/${DOMAIN}/g" | kubectl apply --request-timeout 0 --validate=false -f - && break || sleep 5; done
353-
fi
354-
fi
332+
while true; do kubectl --namespace monitoring apply -f manifests/prometheus-operator --request-timeout 0 && break || sleep 5; done
333+
printf "Waiting for Operator to register third party objects..."
334+
until kubectl --namespace monitoring get servicemonitor > /dev/null 2>&1; do sleep 1; printf "."; done
335+
until kubectl --namespace monitoring get prometheus > /dev/null 2>&1; do sleep 1; printf "."; done
336+
until kubectl --namespace monitoring get alertmanager > /dev/null 2>&1; do sleep 1; printf "."; done
337+
while true; do kubectl --namespace monitoring apply -f manifests/node-exporter --request-timeout 0 && break || sleep 5; done
338+
while true; do kubectl --namespace monitoring apply -f manifests/kube-state-metrics --request-timeout 0 && break || sleep 5; done
339+
while true; do kubectl --namespace monitoring apply -f manifests/grafana/grafana-credentials.yaml --request-timeout 0 && break || sleep 5; done
340+
while true; do kubectl --namespace monitoring apply -f manifests/grafana --request-timeout 0 && break || sleep 5; done
341+
while true; do find manifests/prometheus -type f ! -name prometheus-k8s-roles.yaml ! -name prometheus-k8s-role-bindings.yaml ! -name prometheus-k8s-ingress.yaml -exec kubectl --request-timeout 0 --namespace "monitoring" apply -f {} \; && break || sleep 5; done
342+
while true; do kubectl apply -f manifests/prometheus/prometheus-k8s-roles.yaml --request-timeout 0 && break || sleep 5; done
343+
while true; do kubectl apply -f manifests/prometheus/prometheus-k8s-role-bindings.yaml --request-timeout 0 && break || sleep 5; done
344+
while true; do kubectl --namespace monitoring apply -f manifests/alertmanager/ --request-timeout 0 && break || sleep 5; done
345+
while true; do cat manifests/prometheus/prometheus-k8s-ingress.yaml | sed "s/\${DOMAIN}/${DOMAIN}/g" | kubectl apply --request-timeout 0 --validate=false -f - && break || sleep 5; done
355346
fi
356347
fi
357348

‎manifests/kube-dashboard.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ spec:
3535
spec:
3636
containers:
3737
- name: kubernetes-dashboard
38-
image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.8.0
38+
image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.8.1
3939
ports:
4040
- containerPort: 8443
4141
protocol: TCP

‎manifests/rook/rook-cluster.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,10 @@ spec:
5555
# podAffinity:
5656
# podAntiAffinity:
5757
# tolerations:
58-
storage: # cluster level storage configuration and selection
58+
storage:
5959
useAllNodes: true
6060
useAllDevices: false
61-
deviceFilter:
61+
deviceFilter: sdb
6262
metadataDevice:
6363
location:
6464
storeConfig:

‎manifests/rook/rook-operator.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ spec:
121121
image: rook/rook:master
122122
args: ["operator"]
123123
env:
124+
- name: FLEXVOLUME_DIR_PATH
125+
value: "/var/lib/kubelet/volumeplugins"
124126
- name: ROOK_REPO_PREFIX
125127
value: rook
126128
# The interval to check if every mon is in the quorum.

‎manifests/traefik.yaml

+13-7
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,6 @@ spec:
5757
name: http
5858
- port: 443
5959
name: https
60-
externalIPs:
61-
- ${MASTER_IP}
6260
---
6361
apiVersion: v1
6462
kind: Service
@@ -94,7 +92,7 @@ data:
9492
[entryPoints.https.tls]
9593
[acme]
9694
email = "$EMAIL"
97-
storageFile = "/acme/acme.json"
95+
storage = "/acme/acme.json"
9896
entryPoint = "https"
9997
onDemand = true
10098
onHostRule = true
@@ -107,14 +105,13 @@ data:
107105
Buckets=[0.1,0.3,1.2,5.0]
108106
---
109107
apiVersion: extensions/v1beta1
110-
kind: Deployment
108+
kind: DaemonSet
111109
metadata:
112110
name: traefik-ingress-controller
113111
namespace: kube-system
114112
labels:
115113
k8s-app: traefik-ingress-lb
116114
spec:
117-
replicas: 1
118115
revisionHistoryLimit: 0
119116
template:
120117
metadata:
@@ -129,15 +126,15 @@ spec:
129126
name: traefik-conf
130127
- name: acme
131128
hostPath:
132-
path: /etc/traefik/acme/acme.json
129+
path: /etc/traefik/acme
133130
containers:
134131
- image: traefik
135132
name: traefik-ingress-lb
136133
imagePullPolicy: Always
137134
volumeMounts:
138135
- mountPath: "/config"
139136
name: "config"
140-
- mountPath: "/acme/acme.json"
137+
- mountPath: "/acme"
141138
name: "acme"
142139
ports:
143140
- containerPort: 80
@@ -152,6 +149,15 @@ spec:
152149
- --web.metrics.prometheus.buckets=0.1,0.3,1.2,5.0
153150
- --kubernetes
154151
- --logLevel=DEBUG
152+
affinity:
153+
nodeAffinity:
154+
requiredDuringSchedulingIgnoredDuringExecution:
155+
nodeSelectorTerms:
156+
- matchExpressions:
157+
- key: node-role.kubernetes.io/master
158+
operator: In
159+
values:
160+
-
155161
tolerations:
156162
- key: CriticalAddonsOnly
157163
operator: Exists

0 commit comments

Comments
 (0)