k8s appwrapper config

Author: dgrove-oss

setup.k8s-v1.25/CLUSTER-SETUP.md

@@ -70,13 +70,19 @@ Install the AppWrapper Operator
 ```sh
 kubectl apply --server-side -k setup.k8s-v1.25/appwrapper
 ```
-
-  - the AppWrapper controller is enabled and configured as follows:
-    - `userRBACAdmissionCheck` is disabled,
-    - `schedulerName` is set to `scheduler-plugins-scheduler`,
-    - `queueName` is set to `default-queue`,
-
-TODO: *** UNDER CONSTRUCTION **
+The provided configuration differs from the default configuration of the
+operators as follows:
+- Kubeflow Training Operator:
+  - `gang-scheduler-name` is set to `scheduler-plugins-scheduler`,
+- Kueue:
+  - `manageJobsWithoutQueueName` is enabled,
+  - `batch/job` integration is disabled,
+  - `waitForPodsReady` is disabled,
+- AppWrapper operator:
+  - `userRBACAdmissionCheck` is disabled,
+  - `schedulerName` is set to `scheduler-plugins-scheduler`,
+  - `queueName` is set to `default-queue`,
+- pod priorities, resource requests and limits have been adjusted.
 
 ## Kueue Configuration
 

setup.k8s-v1.25/appwrapper/config_patch.yaml

@@ -0,0 +1,21 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: appwrapper-operator-config
+  namespace: appwrapper-system
+data:
+  config.yaml: |
+    appwrapper:
+      enableKueueIntegrations: true
+      kueueJobReconciller:
+        manageJobsWithoutQueueName: true
+        waitForPodsReady: false
+      queueName: default-queue
+      schedulerName: scheduler-plugins-scheduler
+      userRBACAdmissionCheck: false
+    controllerManager:
+      health:
+        bindAddress: ":8081"
+      metrics:
+        bindAddress: "127.0.0.1:8080"
+      leaderElection: true

setup.k8s-v1.25/appwrapper/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: mlbatch-system
+
+resources:
+- "https://github.com/project-codeflare/appwrapper/config/default?ref=v0.21.0"
+
+images:
+- name: quay.io/ibm/appwrapper
+  newTag: v0.21.0
+
+patches:
+- path: manager_resources_patch.yaml
+- path: config_patch.yaml

setup.k8s-v1.25/appwrapper/manager_resources_patch.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      priorityClassName: system-node-critical
+      containers:
+      - name:  manager
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+          limits:
+            cpu: 1000m
+            memory: 1000Mi

setup.k8s-v1.25/kueue/controller_manager_config.yaml

@@ -22,8 +22,8 @@ clientConnection:
   qps: 50
   burst: 100
 #pprofBindAddress: :8083
-#waitForPodsReady:
-#  enable: false
+waitForPodsReady:
+  enable: false
 #  timeout: 5m
 #  blockAdmission: false
 #  requeuingStrategy:

setup.tmpl/CLUSTER-SETUP.md.tmpl

@@ -148,13 +148,19 @@ Install the AppWrapper Operator
 ```sh
 {{ .KUBECTL }} apply --server-side -k setup.{{ .VERSION }}/appwrapper
 ```
-
-  - the AppWrapper controller is enabled and configured as follows:
-    - `userRBACAdmissionCheck` is disabled,
-    - `schedulerName` is set to `scheduler-plugins-scheduler`,
-    - `queueName` is set to `default-queue`,
-
-TODO: *** UNDER CONSTRUCTION **
+The provided configuration differs from the default configuration of the
+operators as follows:
+- Kubeflow Training Operator:
+  - `gang-scheduler-name` is set to `scheduler-plugins-scheduler`,
+- Kueue:
+  - `manageJobsWithoutQueueName` is enabled,
+  - `batch/job` integration is disabled,
+  - `waitForPodsReady` is disabled,
+- AppWrapper operator:
+  - `userRBACAdmissionCheck` is disabled,
+  - `schedulerName` is set to `scheduler-plugins-scheduler`,
+  - `queueName` is set to `default-queue`,
+- pod priorities, resource requests and limits have been adjusted.
 
 {{- end }}