tensor-pool.yaml

apiVersion: v1
kind: Service
metadata:
  name: condor-head
  labels:
    app: condor-head
spec:
  ports:
  - port: 9618
    name: condor
  clusterIP: None
  selector:
    app: condor-head
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: condor-head
  labels:
    app: condor-head
spec:
  selector:
    matchLabels:
      app: condor-head
  serviceName: "condor"
  template:
    metadata: 
      labels:
        app: condor-head
    spec:
      initContainers:
      - name: init-condor
        image: sfiligoi/prp-htcondor-pool:wn-init
        imagePullPolicy: Always
        resources:
           limits:
             memory: 1Gi
             cpu: 1
           requests:
             memory: 1Gi
             cpu: 1
        volumeMounts:
        - mountPath: /docker-init
          name: docker-init
      containers:
      - name: main
        image: centos:centos7
        resources:  # These resource values should be enough for a small pool
           limits:
             memory: 16Gi
             cpu: 2
           requests:
             memory: 4Gi
             cpu: 1
        command: ["/docker-init/htcondor/start_condor_head.sh"]
        ports:
        - containerPort: 9618
          name: htcondor
        volumeMounts:
        - name: poolpasswd
          mountPath: /etc/wn-init/poolpasswd
          subPath: poolpasswd
          readOnly: true
        - mountPath: /docker-init
          name: docker-init
        - mountPath: /opt/htcondor
          name: condor-bins
        - mountPath: /var/lib/htcondor
          name: condor-state 
      volumes:
      - name: poolpasswd
        secret:
          secretName: condor-poolpassswd
          items:
             - key: poolpasswd
               path: poolpasswd
          defaultMode: 256
      - name: docker-init
        emptyDir: {}
      - name: condor-bins
        emptyDir: {}
      - name: condor-state # Keep this on a truly persistent partition, if you have one
        emptyDir: {}
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: tensor-submit
  labels:
    app: tensor-submit
spec:
  selector:
    matchLabels:
      app: tensor-submit
  serviceName: "condor"
  template:
    metadata:
      labels:
        app: tensor-submit
    spec:
      initContainers:
      - name: init-condor
        image: sfiligoi/prp-htcondor-pool:wn-init
        imagePullPolicy: Always
        resources:
           limits:
             memory: 1Gi
             cpu: 1
           requests:
             memory: 1Gi
             cpu: 1
        volumeMounts:
        - mountPath: /docker-init
          name: docker-init
      containers:
      - name: main
        # You may use any image that you like for interactive use
        image: tensorflow/tensorflow:latest-py3
        resources: # These resource values should be enough for condor itself, but tweak if planning other activity in pool
           limits:
             memory: 16Gi
             cpu: 2
           requests:
             memory: 4Gi
             cpu: 1
        command: ["/docker-init/htcondor/start_condor_submit.sh","condor-head"]
        volumeMounts:
        - name: poolpasswd
          mountPath: /etc/wn-init/poolpasswd
          subPath: poolpasswd
          readOnly: true
        - mountPath: /docker-init
          name: docker-init
        - mountPath: /opt/htcondor
          name: condor-bins
        - mountPath: /var/lib/htcondor
          name: condor-state
      volumes:
      - name: poolpasswd
        secret:
          secretName: condor-poolpassswd
          items:
             - key: poolpasswd
               path: poolpasswd
          defaultMode: 256
      - name: docker-init
        emptyDir: {}
      - name: condor-bins
        emptyDir: {}
      - name: condor-state # Keep this on a truly persistent partition, if you have one
        emptyDir: {}
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tensor-wn
  labels: 
    app: tensor-wn
spec:
  replicas: 3
  selector:
    matchLabels:
      app: tensor-wn
  template:
    metadata: 
      labels:
        app: tensor-wn
    spec:
      initContainers:
      - name: init-condor
        image: sfiligoi/prp-htcondor-pool:wn-init
        imagePullPolicy: Always
        resources:
           limits:
             memory: 1Gi
             cpu: 1
           requests:
             cpu: 1
        volumeMounts:
        - mountPath: /docker-init
          name: docker-init
      containers:
      - name: main
        # You may use any image that is suitable for your apps
        image: tensorflow/tensorflow:latest-gpu-py3
        env:
        - name: OS_IMAGE # This will be advertised in condor, and can be used for matching jobs
          value: "tensorflow/tensorflow:latest-gpu-py3"
        resources: # Tune this to your app needs
           limits:
             nvidia.com/gpu: 1
             memory: 16Gi
             cpu: 2
           requests:
             nvidia.com/gpu: 1
             memory: 4Gi
             cpu: 1
        command: ["/docker-init/htcondor/start_condor_wn.sh","condor-head"]
        #command: ["/bin/sh", "-c","sleep infinity"]
        volumeMounts:
        - name: poolpasswd
          mountPath: /etc/wn-init/poolpasswd
          subPath: poolpasswd
          readOnly: true
        - mountPath: /docker-init
          name: docker-init
        - mountPath: /opt/htcondor
          name: condor-bins
      volumes:
      - name: poolpasswd
        secret:
          secretName: condor-poolpassswd
          items:
             - key: poolpasswd
               path: poolpasswd
          defaultMode: 256
      - name: docker-init
        emptyDir: {}
      - name: condor-bins
        emptyDir: {}