Skip to content

Commit a0ef8db

Browse files
davidweissedavidweisse
committed
nodeinstaller: add nydus-pull container
1 parent 662af0c commit a0ef8db

File tree

13 files changed

+653
-19
lines changed

13 files changed

+653
-19
lines changed

‎.github/workflows/release.yml

+4
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,7 @@ jobs:
251251
serviceMeshImg=$(nix run .#containers.push-service-mesh-proxy -- "$container_registry/contrast/service-mesh-proxy")
252252
tardevSnapshotterImg=$(nix run .#containers.push-tardev-snapshotter -- "$container_registry/contrast/tardev-snapshotter")
253253
nydusSnapshotterImg=$(nix run .#containers.push-nydus-snapshotter -- "$container_registry/contrast/nydus-snapshotter")
254+
nydusPullImg=$(nix run .#containers.push-nydus-pull -- "$container_registry/contrast/nydus-pull")
254255
cryptsetupImg=$(nix run .#containers.push-cryptsetup -- "$container_registry/contrast/cryptsetup")
255256
echo "coordinatorImg=$coordinatorImg" | tee -a "$GITHUB_ENV"
256257
echo "nodeInstallerMsftImg=$nodeInstallerMsftImg" | tee -a "$GITHUB_ENV"
@@ -259,6 +260,7 @@ jobs:
259260
echo "serviceMeshImg=$serviceMeshImg" | tee -a "$GITHUB_ENV"
260261
echo "tardevSnapshotterImg=$tardevSnapshotterImg" | tee -a "$GITHUB_ENV"
261262
echo "nydusSnapshotterImg=$nydusSnapshotterImg" | tee -a "$GITHUB_ENV"
263+
echo "nydusPullImg=$nydusPullImg" | tee -a "$GITHUB_ENV"
262264
echo "cryptsetupImg=$cryptsetupImg" | tee -a "$GITHUB_ENV"
263265
- name: Add tag to Coordinator image
264266
run: |
@@ -272,6 +274,7 @@ jobs:
272274
echo "nodeInstallerKataImgTagged=$(tag "$nodeInstallerKataImg")" | tee -a "$GITHUB_ENV"
273275
echo "initializerImgTagged=$(tag "$initializerImg")" | tee -a "$GITHUB_ENV"
274276
echo "serviceMeshImgTagged=$(tag "$serviceMeshImg")" | tee -a "$GITHUB_ENV"
277+
echo "nydusPullImgTagged=$(tag "$nydusPullImg")" | tee -a "$GITHUB_ENV"
275278
echo "cryptsetupImgTagged=$(tag "$cryptsetupImg")" | tee -a "$GITHUB_ENV"
276279
277280
tardevVer=$(nix eval --impure --raw --expr "(builtins.getFlake \"git+file://$(pwd)?shallow=1\").outputs.legacyPackages.x86_64-linux.microsoft.tardev-snapshotter.version")
@@ -293,6 +296,7 @@ jobs:
293296
echo "ghcr.io/edgelesssys/contrast/node-installer-kata:latest=$nodeInstallerKataImgTagged"
294297
echo "ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest=$tardevSnapshotterImgTagged"
295298
echo "ghcr.io/edgelesssys/contrast/nydus-snapshotter:latest=$nydusSnapshotterImgTagged"
299+
echo "ghcr.io/edgelesssys/contrast/nydus-pull:latest=$nydusPullImgTagged"
296300
echo "ghcr.io/edgelesssys/contrast/cryptsetup:latest=$cryptsetupImgTagged"
297301
} > image-replacements.txt
298302
- name: Upload image replacements file (for main branch PR)

‎go.work

+1
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ use (
44
.
55
./service-mesh
66
./tools/tdx-measure
7+
./tools/nydus-pull
78
)

‎internal/kuberesource/mutators.go

+5
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
applybatchv1 "k8s.io/client-go/applyconfigurations/batch/v1"
1414
applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
1515
applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1"
16+
applyrbacv1 "k8s.io/client-go/applyconfigurations/rbac/v1"
1617
)
1718

1819
const (
@@ -293,6 +294,10 @@ func PatchNamespaces(resources []any, namespace string) []any {
293294
r.Namespace = nsPtr
294295
case *applycorev1.ServiceAccountApplyConfiguration:
295296
r.Namespace = nsPtr
297+
case *applyrbacv1.ClusterRoleBindingApplyConfiguration:
298+
for i := range len(r.Subjects) {
299+
r.Subjects[i].Namespace = nsPtr
300+
}
296301
}
297302
}
298303
return resources

‎internal/kuberesource/parts.go

+89-18
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"k8s.io/apimachinery/pkg/util/intstr"
1616
applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1"
1717
applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
18+
applyrbacv1 "k8s.io/client-go/applyconfigurations/rbac/v1"
1819
)
1920

2021
// ContrastRuntimeClass creates a new RuntimeClassConfig.
@@ -39,6 +40,9 @@ func ContrastRuntimeClass(platform platforms.Platform) (*RuntimeClassConfig, err
3940
// NodeInstallerConfig wraps a DaemonSetApplyConfiguration for a node installer.
4041
type NodeInstallerConfig struct {
4142
*applyappsv1.DaemonSetApplyConfiguration
43+
*applycorev1.ServiceAccountApplyConfiguration
44+
*applyrbacv1.ClusterRoleApplyConfiguration
45+
*applyrbacv1.ClusterRoleBindingApplyConfiguration
4246
}
4347

4448
// NodeInstaller constructs a node installer daemon set.
@@ -118,33 +122,70 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
118122
),
119123
}
120124

125+
nydusPull := Container().
126+
WithName("nydus-pull").
127+
WithImage("ghcr.io/edgelesssys/contrast/nydus-pull:latest").
128+
WithArgs(runtimeHandler).
129+
WithEnv(
130+
EnvVar().
131+
WithName("NODE_NAME").
132+
WithValueFrom(
133+
applycorev1.EnvVarSource().
134+
WithFieldRef(
135+
applycorev1.ObjectFieldSelector().
136+
WithFieldPath("spec.nodeName"),
137+
),
138+
),
139+
).
140+
WithVolumeMounts(
141+
VolumeMount().
142+
WithName("containerd-socket").
143+
WithMountPath("/run/containerd/containerd.sock"),
144+
)
145+
121146
var nodeInstallerImageURL string
122-
var snapshotter *applycorev1.ContainerApplyConfiguration
147+
var containers []*applycorev1.ContainerApplyConfiguration
123148
var snapshotterVolumes []*applycorev1.VolumeApplyConfiguration
124149
switch platform {
125150
case platforms.AKSCloudHypervisorSNP:
126151
nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest"
127-
snapshotter = tardevSnapshotter
152+
containers = append(containers, tardevSnapshotter)
128153
snapshotterVolumes = tardevSnapshotterVolumes
129154
case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX, platforms.MetalQEMUSNPGPU:
130155
nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
131-
snapshotter = nydusSnapshotter
132-
nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
133-
WithName("var-lib-containerd").
134-
WithHostPath(HostPathVolumeSource().
135-
WithPath("/var/lib/containerd").
136-
WithType(corev1.HostPathDirectory),
137-
))
156+
containers = append(containers, nydusSnapshotter, nydusPull)
157+
nydusSnapshotterVolumes = append(nydusSnapshotterVolumes,
158+
Volume().
159+
WithName("var-lib-containerd").
160+
WithHostPath(HostPathVolumeSource().
161+
WithPath("/var/lib/containerd").
162+
WithType(corev1.HostPathDirectory),
163+
),
164+
Volume().
165+
WithName("containerd-socket").
166+
WithHostPath(HostPathVolumeSource().
167+
WithPath("/run/containerd/containerd.sock").
168+
WithType(corev1.HostPathSocket),
169+
),
170+
)
138171
snapshotterVolumes = nydusSnapshotterVolumes
139172
case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.K3sQEMUSNPGPU, platforms.RKE2QEMUTDX:
140173
nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
141-
snapshotter = nydusSnapshotter
142-
nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
143-
WithName("var-lib-containerd").
144-
WithHostPath(HostPathVolumeSource().
145-
WithPath("/var/lib/rancher/k3s/agent/containerd").
146-
WithType(corev1.HostPathDirectory),
147-
))
174+
containers = append(containers, nydusSnapshotter, nydusPull)
175+
nydusSnapshotterVolumes = append(nydusSnapshotterVolumes,
176+
Volume().
177+
WithName("var-lib-containerd").
178+
WithHostPath(HostPathVolumeSource().
179+
WithPath("/var/lib/rancher/k3s/agent/containerd").
180+
WithType(corev1.HostPathDirectory),
181+
),
182+
Volume().
183+
WithName("containerd-socket").
184+
WithHostPath(HostPathVolumeSource().
185+
WithPath("/run/k3s/containerd/containerd.sock").
186+
WithType(corev1.HostPathSocket),
187+
),
188+
)
148189
snapshotterVolumes = nydusSnapshotterVolumes
149190
default:
150191
return nil, fmt.Errorf("unsupported platform %q", platform)
@@ -163,6 +204,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
163204
"contrast.edgeless.systems/platform": platform.String(),
164205
}).
165206
WithSpec(PodSpec().
207+
WithServiceAccountName("nodeinstaller-serviceaccount").
166208
WithHostPID(true).
167209
WithInitContainers(Container().
168210
WithName("installer").
@@ -177,7 +219,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
177219
WithCommand("/bin/node-installer", platform.String()),
178220
).
179221
WithContainers(
180-
snapshotter,
222+
containers...,
181223
).
182224
WithVolumes(append(
183225
snapshotterVolumes,
@@ -193,7 +235,36 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
193235
),
194236
)
195237

196-
return &NodeInstallerConfig{d}, nil
238+
serviceAccount := applycorev1.ServiceAccount("nodeinstaller-serviceaccount", "")
239+
240+
clusterRole := applyrbacv1.ClusterRole("nodeinstaller-clusterrole").
241+
WithRules(
242+
applyrbacv1.PolicyRule().
243+
WithAPIGroups("").
244+
WithResources("pods").
245+
WithVerbs("watch"),
246+
)
247+
248+
clusterRoleBinding := applyrbacv1.ClusterRoleBinding("nodeinstaller-clusterrole-binding").
249+
WithSubjects(
250+
applyrbacv1.Subject().
251+
WithKind("ServiceAccount").
252+
WithName("nodeinstaller-serviceaccount").
253+
WithNamespace(namespace),
254+
).
255+
WithRoleRef(
256+
applyrbacv1.RoleRef().
257+
WithKind("ClusterRole").
258+
WithName("nodeinstaller-clusterrole").
259+
WithAPIGroup("rbac.authorization.k8s.io"),
260+
)
261+
262+
return &NodeInstallerConfig{
263+
DaemonSetApplyConfiguration: d,
264+
ServiceAccountApplyConfiguration: serviceAccount,
265+
ClusterRoleApplyConfiguration: clusterRole,
266+
ClusterRoleBindingApplyConfiguration: clusterRoleBinding,
267+
}, nil
197268
}
198269

199270
// PortForwarderConfig wraps a PodApplyConfiguration for a port forwarder.

‎internal/kuberesource/sets.go

+3
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ func Runtime(platform platforms.Platform) ([]any, error) {
4747
return []any{
4848
runtimeClassApplyConfig,
4949
nodeInstaller.DaemonSetApplyConfiguration,
50+
nodeInstaller.ServiceAccountApplyConfiguration,
51+
nodeInstaller.ClusterRoleApplyConfiguration,
52+
nodeInstaller.ClusterRoleBindingApplyConfiguration,
5053
}, nil
5154
}
5255

‎justfile

+4
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ tardev-snapshotter: (push "tardev-snapshotter")
3333
# Build the nydus-snapshotter, containerize and push it.
3434
nydus-snapshotter: (push "nydus-snapshotter")
3535

36+
# Build the nydus-pull container and push it.
37+
nydus-pull: (push "nydus-pull")
38+
3639
default_cli := "contrast.cli"
3740
default_deploy_target := "openssl"
3841
default_platform := "${default_platform}"
@@ -49,6 +52,7 @@ node-installer platform=default_platform:
4952
;;
5053
"Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"Metal-QEMU-SNP-GPU"|"K3s-QEMU-SNP"|"K3s-QEMU-SNP-GPU"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
5154
just push "nydus-snapshotter"
55+
just push "nydus-pull"
5256
just push "node-installer-kata"
5357
;;
5458
"AKS-PEER-SNP")

‎packages/by-name/contrast/package.nix

+1-1
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ buildGoModule rec {
176176
(fileset.difference (fileset.fileFilter (file: hasSuffix ".go" file.name) root) (
177177
fileset.unions [
178178
(path.append root "service-mesh")
179-
(path.append root "tools/tdx-measure")
179+
(path.append root "tools")
180180
]
181181
))
182182
];

‎packages/by-name/nydus-pull/package.nix

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright 2024 Edgeless Systems GmbH
2+
# SPDX-License-Identifier: AGPL-3.0-only
3+
4+
{ buildGoModule }:
5+
6+
buildGoModule rec {
7+
pname = "nydus-pull";
8+
version = builtins.readFile ../../../version.txt;
9+
10+
src = ../../../tools/nydus-pull;
11+
12+
proxyVendor = true;
13+
vendorHash = "sha256-bzCdcDfdivf52CerJ+9Nf5i+/laqjBWKNhhyLS8eBs4=";
14+
15+
subPackages = [ "." ];
16+
17+
CGO_ENABLED = 0;
18+
ldflags = [
19+
"-s"
20+
"-X main.version=v${version}"
21+
];
22+
23+
preCheck = ''
24+
export CGO_ENABLED=1
25+
'';
26+
27+
checkPhase = ''
28+
runHook preCheck
29+
go test -race ./...
30+
runHook postCheck
31+
'';
32+
33+
meta.mainProgram = "nydus-pull";
34+
}

‎packages/containers.nix

+9
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,15 @@ let
178178
Cmd = [ "${lib.getExe pkgs.cloud-api-adaptor.entrypoint}" ];
179179
};
180180
};
181+
182+
nydus-pull = dockerTools.buildImage {
183+
name = "nydus-pull";
184+
tag = "v${pkgs.nydus-pull.version}";
185+
copyToRoot = with dockerTools; [ caCertificates ];
186+
config = {
187+
Entrypoint = [ "${lib.getExe pkgs.nydus-pull}" ];
188+
};
189+
};
181190
};
182191
in
183192
containers

‎packages/scripts.nix

+2
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@
122122
nix-update --version=skip --flake legacyPackages.x86_64-linux.service-mesh
123123
echo "Updating vendorHash of contrast.cli package" >&2
124124
nix-update --version=skip --flake legacyPackages.x86_64-linux.contrast
125+
echo "Updating vendorHash of nydus-pull package" >&2
126+
nix-update --version=skip --flake legacyPackages.x86_64-linux.nydus-pull
125127
126128
echo "Updating src hash of kata.kata-kernel-uvm.configfile" >&2
127129
nix-update --version=skip --flake legacyPackages.x86_64-linux.kata.kata-kernel-uvm.configfile

0 commit comments

Comments
 (0)