Skip to content

Commit 2cd1e94

Browse files
upodroidupodroid
committed
cloud init isn't widely available on other GCE OS, default to using startup-script unless its an Ubuntu/COS OS
1 parent 3440db0 commit 2cd1e94

File tree

5 files changed

+328
-35
lines changed

5 files changed

+328
-35
lines changed

‎pkg/model/gcemodel/autoscalinggroup.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,16 @@ func (b *AutoscalingGroupModelBuilder) buildInstanceTemplate(c *fi.CloudupModelB
132132
}
133133

134134
if startupScript != nil {
135-
if !fi.ValueOf(b.Cluster.Spec.CloudProvider.GCE.UseStartupScript) {
136-
// Use "user-data" instead of "startup-script", for compatibility with cloud-init
135+
// GCE doesn't bundle cloud-init on every OS unless cloud-init is present in the upstream distribution
136+
// So far, thats only true for COS and Ubuntu
137+
switch {
138+
case fi.ValueOf(b.Cluster.Spec.CloudProvider.GCE.UseStartupScript):
139+
t.Metadata["startup-script"] = startupScript
140+
case strings.HasPrefix(ig.Spec.Image, "cos-cloud/"):
141+
t.Metadata["user-data"] = startupScript
142+
case strings.HasPrefix(ig.Spec.Image, "ubuntu-os-cloud/"):
137143
t.Metadata["user-data"] = startupScript
138-
} else {
144+
default:
139145
t.Metadata["startup-script"] = startupScript
140146
}
141147
}

‎tests/integration/update_cluster/many-addons-gce/data/google_compute_instance_template_master-us-test1-a-minimal-example-com_metadata_startup-script‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
#!/bin/bash
2+
set -o errexit
3+
set -o nounset
4+
set -o pipefail
5+
6+
NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-amd64
7+
NODEUP_HASH_AMD64=c86e072f622b91546b7b3f3cb1a0f8a131e48b966ad018a0ac1520ceedf37725
8+
NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-arm64
9+
NODEUP_HASH_ARM64=64a9a9510538a449e85d05e13e3cd98b80377d68a673447c26821d40f00f0075
10+
11+
12+
13+
14+
15+
sysctl -w net.core.rmem_max=16777216 || true
16+
sysctl -w net.core.wmem_max=16777216 || true
17+
sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
18+
sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
19+
20+
21+
function ensure-install-dir() {
22+
INSTALL_DIR="/opt/kops"
23+
# On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
24+
if [[ -d /var/lib/toolbox ]]; then
25+
INSTALL_DIR="/var/lib/toolbox/kops"
26+
fi
27+
mkdir -p ${INSTALL_DIR}/bin
28+
mkdir -p ${INSTALL_DIR}/conf
29+
cd ${INSTALL_DIR}
30+
}
31+
32+
# Retry a download until we get it. args: name, sha, urls
33+
download-or-bust() {
34+
echo "== Downloading $1 with hash $2 from $3 =="
35+
local -r file="$1"
36+
local -r hash="$2"
37+
local -a urls
38+
IFS=, read -r -a urls <<< "$3"
39+
40+
if [[ -f "${file}" ]]; then
41+
if ! validate-hash "${file}" "${hash}"; then
42+
rm -f "${file}"
43+
else
44+
return 0
45+
fi
46+
fi
47+
48+
while true; do
49+
for url in "${urls[@]}"; do
50+
commands=(
51+
"curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
52+
"wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10"
53+
"curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
54+
"wget -O ${file} --connect-timeout=20 --tries=6 --wait=10"
55+
)
56+
for cmd in "${commands[@]}"; do
57+
echo "== Downloading ${url} using ${cmd} =="
58+
if ! (${cmd} "${url}"); then
59+
echo "== Failed to download ${url} using ${cmd} =="
60+
continue
61+
fi
62+
if ! validate-hash "${file}" "${hash}"; then
63+
echo "== Failed to validate hash for ${url} =="
64+
rm -f "${file}"
65+
else
66+
echo "== Downloaded ${url} with hash ${hash} =="
67+
return 0
68+
fi
69+
done
70+
done
71+
72+
echo "== All downloads failed; sleeping before retrying =="
73+
sleep 60
74+
done
75+
}
76+
77+
validate-hash() {
78+
local -r file="$1"
79+
local -r expected="$2"
80+
local actual
81+
82+
actual=$(sha256sum "${file}" | awk '{ print $1 }') || true
83+
if [[ "${actual}" != "${expected}" ]]; then
84+
echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} =="
85+
return 1
86+
fi
87+
}
88+
89+
function download-release() {
90+
case "$(uname -m)" in
91+
x86_64*|i?86_64*|amd64*)
92+
NODEUP_URL="${NODEUP_URL_AMD64}"
93+
NODEUP_HASH="${NODEUP_HASH_AMD64}"
94+
;;
95+
aarch64*|arm64*)
96+
NODEUP_URL="${NODEUP_URL_ARM64}"
97+
NODEUP_HASH="${NODEUP_HASH_ARM64}"
98+
;;
99+
*)
100+
echo "Unsupported host arch: $(uname -m)" >&2
101+
exit 1
102+
;;
103+
esac
104+
105+
cd ${INSTALL_DIR}/bin
106+
download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
107+
108+
chmod +x nodeup
109+
110+
echo "== Running nodeup =="
111+
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
112+
( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 )
113+
}
114+
115+
####################################################################################
116+
117+
/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured =="
118+
119+
echo "== nodeup node config starting =="
120+
ensure-install-dir
121+
122+
cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
123+
CloudProvider: gce
124+
ClusterName: minimal.example.com
125+
ConfigBase: memfs://tests/minimal.example.com
126+
InstanceGroupName: master-us-test1-a
127+
InstanceGroupRole: ControlPlane
128+
NodeupConfigHash: zvt1dlE0mlG53w80tjMqIyCE6f8pEzREEoTLZA1Chhw=
129+
130+
__EOF_KUBE_ENV
131+
132+
download-release
133+
echo "== nodeup node config done =="

‎tests/integration/update_cluster/many-addons-gce/data/google_compute_instance_template_nodes-minimal-example-com_metadata_startup-script‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
#!/bin/bash
2+
set -o errexit
3+
set -o nounset
4+
set -o pipefail
5+
6+
NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-amd64
7+
NODEUP_HASH_AMD64=c86e072f622b91546b7b3f3cb1a0f8a131e48b966ad018a0ac1520ceedf37725
8+
NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.34.0-beta.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.34.0-beta.1/nodeup-linux-arm64
9+
NODEUP_HASH_ARM64=64a9a9510538a449e85d05e13e3cd98b80377d68a673447c26821d40f00f0075
10+
11+
12+
13+
14+
15+
sysctl -w net.core.rmem_max=16777216 || true
16+
sysctl -w net.core.wmem_max=16777216 || true
17+
sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
18+
sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
19+
20+
21+
function ensure-install-dir() {
22+
INSTALL_DIR="/opt/kops"
23+
# On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
24+
if [[ -d /var/lib/toolbox ]]; then
25+
INSTALL_DIR="/var/lib/toolbox/kops"
26+
fi
27+
mkdir -p ${INSTALL_DIR}/bin
28+
mkdir -p ${INSTALL_DIR}/conf
29+
cd ${INSTALL_DIR}
30+
}
31+
32+
# Retry a download until we get it. args: name, sha, urls
33+
download-or-bust() {
34+
echo "== Downloading $1 with hash $2 from $3 =="
35+
local -r file="$1"
36+
local -r hash="$2"
37+
local -a urls
38+
IFS=, read -r -a urls <<< "$3"
39+
40+
if [[ -f "${file}" ]]; then
41+
if ! validate-hash "${file}" "${hash}"; then
42+
rm -f "${file}"
43+
else
44+
return 0
45+
fi
46+
fi
47+
48+
while true; do
49+
for url in "${urls[@]}"; do
50+
commands=(
51+
"curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
52+
"wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10"
53+
"curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
54+
"wget -O ${file} --connect-timeout=20 --tries=6 --wait=10"
55+
)
56+
for cmd in "${commands[@]}"; do
57+
echo "== Downloading ${url} using ${cmd} =="
58+
if ! (${cmd} "${url}"); then
59+
echo "== Failed to download ${url} using ${cmd} =="
60+
continue
61+
fi
62+
if ! validate-hash "${file}" "${hash}"; then
63+
echo "== Failed to validate hash for ${url} =="
64+
rm -f "${file}"
65+
else
66+
echo "== Downloaded ${url} with hash ${hash} =="
67+
return 0
68+
fi
69+
done
70+
done
71+
72+
echo "== All downloads failed; sleeping before retrying =="
73+
sleep 60
74+
done
75+
}
76+
77+
validate-hash() {
78+
local -r file="$1"
79+
local -r expected="$2"
80+
local actual
81+
82+
actual=$(sha256sum "${file}" | awk '{ print $1 }') || true
83+
if [[ "${actual}" != "${expected}" ]]; then
84+
echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} =="
85+
return 1
86+
fi
87+
}
88+
89+
function download-release() {
90+
case "$(uname -m)" in
91+
x86_64*|i?86_64*|amd64*)
92+
NODEUP_URL="${NODEUP_URL_AMD64}"
93+
NODEUP_HASH="${NODEUP_HASH_AMD64}"
94+
;;
95+
aarch64*|arm64*)
96+
NODEUP_URL="${NODEUP_URL_ARM64}"
97+
NODEUP_HASH="${NODEUP_HASH_ARM64}"
98+
;;
99+
*)
100+
echo "Unsupported host arch: $(uname -m)" >&2
101+
exit 1
102+
;;
103+
esac
104+
105+
cd ${INSTALL_DIR}/bin
106+
download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
107+
108+
chmod +x nodeup
109+
110+
echo "== Running nodeup =="
111+
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
112+
( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 )
113+
}
114+
115+
####################################################################################
116+
117+
/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured =="
118+
119+
echo "== nodeup node config starting =="
120+
ensure-install-dir
121+
122+
cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
123+
CloudProvider: gce
124+
ClusterName: minimal.example.com
125+
ConfigServer:
126+
CACertificates: |
127+
-----BEGIN CERTIFICATE-----
128+
MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
129+
BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
130+
ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
131+
SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
132+
jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
133+
MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
134+
MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
135+
tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
136+
-----END CERTIFICATE-----
137+
-----BEGIN CERTIFICATE-----
138+
MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
139+
BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
140+
OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
141+
SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
142+
WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
143+
MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
144+
MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
145+
9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
146+
-----END CERTIFICATE-----
147+
servers:
148+
- https://kops-controller.internal.minimal.example.com:3988/
149+
InstanceGroupName: nodes
150+
InstanceGroupRole: Node
151+
NodeupConfigHash: +QsrBzqxqccUBVMNc3E1kf3Ry1aCSFqc9zbYtLo2SUU=
152+
153+
__EOF_KUBE_ENV
154+
155+
download-release
156+
echo "== nodeup node config done =="

0 commit comments

Comments
 (0)