Skip to content

Commit 6c5d7ba

Browse files
yetoneyetone
committed
feat: restricted security context
1 parent 4dc695c commit 6c5d7ba

File tree

11 files changed

+112
-29
lines changed

11 files changed

+112
-29
lines changed

bento-downloader/Dockerfile renamed to container_images/bento-downloader/Dockerfile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,13 @@ RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud
1111
&& tar -xf google-cloud-cli-410.tar.gz \
1212
&& ./google-cloud-sdk/install.sh \
1313
&& rm google-cloud-cli-410.tar.gz
14+
15+
ARG USERNAME=yetone
16+
ARG USER_UID=1000
17+
ARG USER_GID=$USER_UID
18+
19+
# Create the user
20+
RUN groupadd --gid $USER_GID $USERNAME \
21+
&& useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
22+
23+
USER $USER_UID:$USER_GID

bento-downloader/Makefile renamed to container_images/bento-downloader/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
IMAGE := quay.io/bentoml/bento-downloader:0.0.1
1+
IMAGE := quay.io/bentoml/bento-downloader:0.0.3
22

33
build:
44
docker build -t ${IMAGE} .

‎container_images/buildah/Dockerfile‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
FROM quay.io/buildah/stable:v1.23.1
2+
3+
ARG BENTO_USER_UID=1034
4+
ARG BENTO_USER_GID=1034
5+
6+
RUN touch /etc/subgid /etc/subuid \
7+
&& chmod g=u /etc/subgid /etc/subuid /etc/passwd \
8+
&& echo build:$BENTO_USER_UID:$BENTO_USER_GID > /etc/subuid \
9+
&& echo build:$BENTO_USER_UID:$BENTO_USER_GID > /etc/subgid
10+
11+
# Use chroot since the default runc does not work when running rootless
12+
RUN echo "export BUILDAH_ISOLATION=chroot" >> /home/build/.bashrc
13+
14+
USER $BENTO_USER_UID:$BENTO_USER_GID
15+
WORKDIR /home/build
16+

‎container_images/buildah/Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
IMAGE := quay.io/bentoml/bentoml-buildah:0.0.1
2+
3+
build:
4+
docker build -t ${IMAGE} .
5+
docker push ${IMAGE}

‎controllers/resources/bentorequest_controller.go‎

Lines changed: 57 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -625,12 +625,18 @@ const (
625625
BentoImageBuildEngineKaniko BentoImageBuildEngine = "kaniko"
626626
BentoImageBuildEngineBuildkit BentoImageBuildEngine = "buildkit"
627627
BentoImageBuildEngineBuildkitRootless BentoImageBuildEngine = "buildkit-rootless"
628+
BentoImageBuildEngineBuildah BentoImageBuildEngine = "buildah"
628629
)
629630

630631
const (
631632
EnvBentoImageBuildEngine = "BENTO_IMAGE_BUILD_ENGINE"
633+
EnvRunInOpenshift = "RUN_IN_OPENSHIFT"
632634
)
633635

636+
func checkIfRunInOpenshift() bool {
637+
return os.Getenv(EnvRunInOpenshift) == commonconsts.KubeLabelValueTrue
638+
}
639+
634640
func getBentoImageBuildEngine() BentoImageBuildEngine {
635641
engine := os.Getenv(EnvBentoImageBuildEngine)
636642
if engine == "" {
@@ -1119,8 +1125,10 @@ func (r *BentoRequestReconciler) generateImageBuilderPod(ctx context.Context, op
11191125
logrus.Infof("Image builder is using the images %v", *internalImages)
11201126

11211127
buildEngine := getBentoImageBuildEngine()
1128+
isRunInOpenshift := checkIfRunInOpenshift()
11221129

1123-
privileged := buildEngine != BentoImageBuildEngineBuildkitRootless
1130+
privileged := buildEngine != BentoImageBuildEngineBuildkitRootless || isRunInOpenshift
1131+
unprivilegedUID := int64(1034)
11241132

11251133
bentoDownloadCommandTemplate, err := template.New("downloadCommand").Parse(`
11261134
set -e
@@ -1141,7 +1149,7 @@ echo "Removing bento tar file..."
11411149
rm /tmp/downloaded.tar
11421150
{{if not .Privileged}}
11431151
echo "Changing directory permission..."
1144-
chown -R 1000:1000 /workspace
1152+
chown -R {{ .UnprivilegedUID }}:{{ .UnprivilegedUID }} /workspace
11451153
{{end}}
11461154
echo "Done"
11471155
`)
@@ -1159,6 +1167,7 @@ echo "Done"
11591167
"BentoRepositoryName": bentoRepositoryName,
11601168
"BentoVersion": bentoVersion,
11611169
"Privileged": privileged,
1170+
"UnprivilegedUID": unprivilegedUID,
11621171
})
11631172
if err != nil {
11641173
err = errors.Wrap(err, "failed to execute download command template")
@@ -1200,6 +1209,17 @@ echo "Done"
12001209
})
12011210
}
12021211

1212+
restrictedSecurityContext := &corev1.SecurityContext{
1213+
AllowPrivilegeEscalation: pointer.BoolPtr(false),
1214+
RunAsNonRoot: pointer.BoolPtr(true),
1215+
SeccompProfile: &corev1.SeccompProfile{
1216+
Type: corev1.SeccompProfileTypeRuntimeDefault,
1217+
},
1218+
Capabilities: &corev1.Capabilities{
1219+
Drop: []corev1.Capability{"ALL"},
1220+
},
1221+
}
1222+
12031223
initContainers := []corev1.Container{
12041224
{
12051225
Name: "bento-downloader",
@@ -1209,9 +1229,10 @@ echo "Done"
12091229
"-c",
12101230
bentoDownloadCommand,
12111231
},
1212-
VolumeMounts: volumeMounts,
1213-
Resources: downloaderContainerResources,
1214-
EnvFrom: downloaderContainerEnvFrom,
1232+
VolumeMounts: volumeMounts,
1233+
Resources: downloaderContainerResources,
1234+
EnvFrom: downloaderContainerEnvFrom,
1235+
SecurityContext: restrictedSecurityContext,
12151236
},
12161237
}
12171238

@@ -1304,7 +1325,7 @@ echo "Removing model tar file..."
13041325
rm /tmp/downloaded.tar
13051326
{{if not .Privileged}}
13061327
echo "Changing directory permission..."
1307-
chown -R 1000:1000 /workspace
1328+
chown -R {{ .UnprivilegedUID }}:{{ .UnprivilegedUID }} /workspace
13081329
{{end}}
13091330
echo "Done"
13101331
`)).Execute(&modelDownloadCommandOutput, map[string]interface{}{
@@ -1315,6 +1336,7 @@ echo "Done"
13151336
"ModelRepositoryName": modelRepositoryName,
13161337
"ModelVersion": modelVersion,
13171338
"Privileged": privileged,
1339+
"UnprivilegedUID": unprivilegedUID,
13181340
})
13191341
if err != nil {
13201342
err = errors.Wrap(err, "failed to generate download command")
@@ -1329,9 +1351,10 @@ echo "Done"
13291351
"-c",
13301352
modelDownloadCommand,
13311353
},
1332-
VolumeMounts: volumeMounts,
1333-
Resources: downloaderContainerResources,
1334-
EnvFrom: downloaderContainerEnvFrom,
1354+
VolumeMounts: volumeMounts,
1355+
Resources: downloaderContainerResources,
1356+
EnvFrom: downloaderContainerEnvFrom,
1357+
SecurityContext: restrictedSecurityContext,
13351358
})
13361359
}
13371360

@@ -1459,6 +1482,8 @@ echo "Done"
14591482
builderImage = internalImages.Buildkit
14601483
case BentoImageBuildEngineBuildkitRootless:
14611484
builderImage = internalImages.BuildkitRootless
1485+
case BentoImageBuildEngineBuildah:
1486+
builderImage = internalImages.Buildah
14621487
default:
14631488
err = errors.Errorf("unknown bento image build engine %s", buildEngine)
14641489
return
@@ -1483,18 +1508,33 @@ echo "Done"
14831508

14841509
var builderContainerSecurityContext *corev1.SecurityContext
14851510

1511+
//nolint: gocritic
14861512
if buildEngine == BentoImageBuildEngineBuildkit {
14871513
builderContainerSecurityContext = &corev1.SecurityContext{
14881514
Privileged: pointer.BoolPtr(true),
14891515
}
14901516
} else if buildEngine == BentoImageBuildEngineBuildkitRootless {
14911517
kubeAnnotations["container.apparmor.security.beta.kubernetes.io/builder"] = "unconfined"
1518+
for _, container := range initContainers {
1519+
kubeAnnotations[fmt.Sprintf("container.apparmor.security.beta.kubernetes.io/%s", container.Name)] = "unconfined"
1520+
}
14921521
builderContainerSecurityContext = &corev1.SecurityContext{
14931522
SeccompProfile: &corev1.SeccompProfile{
14941523
Type: corev1.SeccompProfileTypeUnconfined,
14951524
},
1496-
RunAsUser: pointer.Int64Ptr(1000),
1497-
RunAsGroup: pointer.Int64Ptr(1000),
1525+
RunAsUser: pointer.Int64Ptr(unprivilegedUID),
1526+
RunAsGroup: pointer.Int64Ptr(unprivilegedUID),
1527+
}
1528+
} else if buildEngine == BentoImageBuildEngineBuildah {
1529+
kubeAnnotations["openshift.io/scc"] = "anyuid"
1530+
builderContainerSecurityContext = &corev1.SecurityContext{
1531+
RunAsUser: pointer.Int64Ptr(unprivilegedUID),
1532+
RunAsGroup: pointer.Int64Ptr(unprivilegedUID),
1533+
Capabilities: &corev1.Capabilities{
1534+
Drop: []corev1.Capability{
1535+
"KILL",
1536+
},
1537+
},
14981538
}
14991539
}
15001540

@@ -1614,6 +1654,12 @@ echo "Done"
16141654
Containers: []corev1.Container{
16151655
container,
16161656
},
1657+
SecurityContext: &corev1.PodSecurityContext{
1658+
RunAsNonRoot: pointer.BoolPtr(true),
1659+
SeccompProfile: &corev1.SeccompProfile{
1660+
Type: corev1.SeccompProfileTypeRuntimeDefault,
1661+
},
1662+
},
16171663
},
16181664
}
16191665

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.18
44

55
require (
66
github.com/aws/aws-sdk-go v1.44.152
7-
github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1
7+
github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d
88
github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451
99
github.com/huandu/xstrings v1.3.2
1010
github.com/iancoleman/strcase v0.2.0

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ github.com/aws/aws-sdk-go v1.44.152 h1:L9aaepO8wHB67gwuGD8VgIYH/cmQDxieCt7FeLa0+
7777
github.com/aws/aws-sdk-go v1.44.152/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
7878
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
7979
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
80-
github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1 h1:VgN2DLopHoMaEVOiD8J4bZO0L1BGXLRSmNSs7GkfbHo=
81-
github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ=
80+
github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d h1:r+iumKOD+Ri4u2NggQYudjQurIYN2Rs0nOW48faSt9E=
81+
github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ=
8282
github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451 h1:FNxCbN61Ev8ea6BXzlfmRUT5CYNmqlOv8zDRGs8ufVE=
8383
github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451/go.mod h1:q7tt064G8YIiAwQabKyVaKEdSIHYDQA9Oyt+kyCsflU=
8484
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

‎helm/yatai-image-builder/templates/secret-env.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,13 @@ stringData:
3232
INTERNAL_IMAGES_KANIKO: {{ .Values.internalImages.kaniko | quote }}
3333
INTERNAL_IMAGES_BUILDKIT: {{ .Values.internalImages.buildkit | quote }}
3434
INTERNAL_IMAGES_BUILDKIT_ROOTLESS: {{ .Values.internalImages.buildkitRootless | quote }}
35+
INTERNAL_IMAGES_BUILDAH: {{ .Values.internalImages.buildah | quote }}
3536

3637
{{- if .Values.dockerRegistry.useAWSECRWithIAMRole }}
3738
AWS_ECR_WITH_IAM_ROLE: "true"
3839
AWS_ECR_REGION: {{ .Values.dockerRegistry.awsECRRegion | quote }}
3940
{{- end }}
4041

4142
BENTO_IMAGE_BUILD_ENGINE: {{ .Values.bentoImageBuildEngine | quote }}
43+
44+
RUN_IN_OPENSHIFT: {{ .Values.runInOpenshift | quote }}

‎helm/yatai-image-builder/values.yaml‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,16 @@ serviceAccount:
2727

2828
podAnnotations: {}
2929

30-
podSecurityContext: {}
31-
# fsGroup: 2000
30+
podSecurityContext:
31+
runAsNonRoot: true
32+
seccompProfile:
33+
type: RuntimeDefault
3234

33-
securityContext: {}
34-
# capabilities:
35-
# drop:
36-
# - ALL
37-
# readOnlyRootFilesystem: true
38-
# runAsNonRoot: true
39-
# runAsUser: 1000
35+
securityContext:
36+
allowPrivilegeEscalation: false
37+
capabilities:
38+
drop:
39+
- ALL
4040

4141
service:
4242
type: ClusterIP
@@ -96,9 +96,11 @@ aws:
9696
secretAccessKeyExistingSecretKey: ''
9797

9898
internalImages:
99-
bentoDownloader: quay.io/bentoml/bento-downloader:0.0.1
99+
bentoDownloader: quay.io/bentoml/bento-downloader:0.0.3
100100
kaniko: quay.io/bentoml/kaniko:1.9.1
101101
buildkit: quay.io/bentoml/buildkit:master
102102
buildkitRootless: quay.io/bentoml/buildkit:master-rootless
103+
buildah: quay.io/bentoml/bentoml-buildah:0.0.1
103104

104105
bentoImageBuildEngine: kaniko # options: kaniko, buildkit, buildkit-rootless
106+
runInOpenshift: false

‎scripts/quick-install-yatai-image-builder.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,8 @@ if [ "${USE_LOCAL_HELM_CHART}" = "true" ]; then
226226
--set aws.accessKeyID=${AWS_ACCESS_KEY_ID} \
227227
--set aws.secretAccessKeyExistingSecretName=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME} \
228228
--set aws.secretAccessKeyExistingSecretKey=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY} \
229-
--set bentoImageBuildEngine=buildkit-rootless
229+
--set bentoImageBuildEngine=buildah \
230+
--set runInOpenshift=true
230231
else
231232
helm_repo_name=bentoml
232233
helm_repo_url=https://bentoml.github.io/helm-charts

0 commit comments

Comments
 (0)