From cd5320cda22cc21fe8c44e558aa0e3ebf8df7307 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Mon, 27 Jul 2020 15:16:17 -0400 Subject: [PATCH 01/18] docker login on invoker pods when using private registry --- .../configMapFiles/dockerLogin/dockerLogin.sh | 1 + helm/openwhisk/templates/_helpers.tpl | 5 ++++ helm/openwhisk/templates/invoker-cm.yaml | 1 + helm/openwhisk/templates/invoker-pod.yaml | 26 ++++++++++++++++--- 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100755 helm/openwhisk/configMapFiles/dockerLogin/dockerLogin.sh diff --git a/helm/openwhisk/configMapFiles/dockerLogin/dockerLogin.sh b/helm/openwhisk/configMapFiles/dockerLogin/dockerLogin.sh new file mode 100755 index 00000000..42682b65 --- /dev/null +++ b/helm/openwhisk/configMapFiles/dockerLogin/dockerLogin.sh @@ -0,0 +1 @@ +docker login $RUNTIMES_REGISTRY -u $RUNTIMES_REGISTRY_USERNAME -p $RUNTIMES_REGISTRY_PASSWORD diff --git a/helm/openwhisk/templates/_helpers.tpl b/helm/openwhisk/templates/_helpers.tpl index f57c5781..108f87b1 100644 --- a/helm/openwhisk/templates/_helpers.tpl +++ b/helm/openwhisk/templates/_helpers.tpl @@ -310,6 +310,11 @@ imagePullSecrets: mountPath: "/invoker-scripts/configureDNS.sh" subPath: "configureDNS.sh" {{- end }} +{{- if ne .Values.docker.registry.name "" }} + - name: scripts-dir + mountPath: "/invoker-scripts/dockerLogin.sh" + subPath: "dockerLogin.sh" +{{- end }} {{- end }} {{- end }} diff --git a/helm/openwhisk/templates/invoker-cm.yaml b/helm/openwhisk/templates/invoker-cm.yaml index a09f5075..0c689a31 100644 --- a/helm/openwhisk/templates/invoker-cm.yaml +++ b/helm/openwhisk/templates/invoker-cm.yaml @@ -24,3 +24,4 @@ metadata: data: {{ (.Files.Glob "configMapFiles/dockerPullRuntimes/playbook.yml").AsConfig | indent 2 }} {{ (.Files.Glob "configMapFiles/configureDNS/configureDNS.sh").AsConfig | indent 2 }} +{{ (.Files.Glob "configMapFiles/dockerLogin/dockerLogin.sh").AsConfig | indent 2 }} diff --git a/helm/openwhisk/templates/invoker-pod.yaml b/helm/openwhisk/templates/invoker-pod.yaml index dbb80289..6cc37b8b 100644 --- a/helm/openwhisk/templates/invoker-pod.yaml +++ b/helm/openwhisk/templates/invoker-pod.yaml @@ -83,11 +83,17 @@ spec: - name: invoker image: "{{- .Values.docker.registry.name -}}{{- .Values.invoker.imageName -}}:{{- .Values.invoker.imageTag -}}" imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }} + command: + - /bin/bash + - -c + - +{{- if ne .Values.docker.registry.name "" }} + . /invoker-scripts/dockerLogin.sh && +{{- end }} {{- if and (eq .Values.invoker.containerFactory.impl "docker") .Values.invoker.containerFactory.networkConfig.dns.inheritInvokerConfig }} - command: [ "/bin/bash", "-c", ". /invoker-scripts/configureDNS.sh && /init.sh --uniqueName $INVOKER_NAME" ] -{{- else }} - command: [ "/bin/bash", "-c", "/init.sh --uniqueName $INVOKER_NAME" ] + . /invoker-scripts/configureDNS.sh && {{- end }} + /init.sh --uniqueName $INVOKER_NAME env: - name: "PORT" value: {{ .Values.invoker.port | quote }} @@ -211,6 +217,20 @@ spec: value: "{{ .Values.akka.actorSystemTerminateTimeout }}" - name: "CONFIG_whisk_runtime_delete_timeout" value: "{{ .Values.invoker.runtimeDeleteTimeout }}" +{{- if ne .Values.docker.registry.name "" }} + - name: "RUNTIMES_REGISTRY" + value: "{{- .Values.docker.registry.name -}}" + - name: "RUNTIMES_REGISTRY_USERNAME" + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-docker.registry.auth + key: docker_registry_username + - name: "RUNTIMES_REGISTRY_PASSWORD" + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-docker.registry.auth + key: docker_registry_password +{{- end }} ports: - name: invoker containerPort: {{ .Values.invoker.port }} From e1ffcb0538ae4315efa6f676c7f40c4bb2c48907 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Mon, 4 Jan 2021 16:10:04 -0500 Subject: [PATCH 02/18] support resources definitions on most pods --- helm/openwhisk/templates/apigateway-pod.yaml | 4 ++++ helm/openwhisk/templates/controller-pod.yaml | 4 ++++ helm/openwhisk/templates/invoker-pod.yaml | 4 ++++ helm/openwhisk/templates/kafka-pod.yaml | 4 ++++ helm/openwhisk/templates/nginx-pod.yaml | 4 ++++ helm/openwhisk/templates/redis-pod.yaml | 4 ++++ helm/openwhisk/templates/zookeeper-pod.yaml | 5 ++++- helm/openwhisk/values.yaml | 7 +++++++ 8 files changed, 35 insertions(+), 1 deletion(-) diff --git a/helm/openwhisk/templates/apigateway-pod.yaml b/helm/openwhisk/templates/apigateway-pod.yaml index 13b3ac3c..62cfd550 100644 --- a/helm/openwhisk/templates/apigateway-pod.yaml +++ b/helm/openwhisk/templates/apigateway-pod.yaml @@ -56,6 +56,10 @@ spec: containerPort: {{ .Values.apigw.mgmtPort }} - name: api containerPort: {{ .Values.apigw.apiPort }} +{{- if .Values.apigw.resources }} + resources: +{{ toYaml .Values.apigw.resources | indent 12 }} +{{- end }} env: - name: "REDIS_HOST" value: "{{ include "openwhisk.redis_host" . }}" diff --git a/helm/openwhisk/templates/controller-pod.yaml b/helm/openwhisk/templates/controller-pod.yaml index 28a97565..0a18496f 100644 --- a/helm/openwhisk/templates/controller-pod.yaml +++ b/helm/openwhisk/templates/controller-pod.yaml @@ -107,6 +107,10 @@ spec: initialDelaySeconds: {{ .Values.probes.controller.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.probes.controller.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.probes.controller.readinessProbe.timeoutSeconds }} +{{- if .Values.controller.resources }} + resources: +{{ toYaml .Values.controller.resources | indent 10 }} +{{- end }} env: - name: "PORT" value: {{ .Values.controller.port | quote }} diff --git a/helm/openwhisk/templates/invoker-pod.yaml b/helm/openwhisk/templates/invoker-pod.yaml index 6cc37b8b..e84b5b25 100644 --- a/helm/openwhisk/templates/invoker-pod.yaml +++ b/helm/openwhisk/templates/invoker-pod.yaml @@ -94,6 +94,10 @@ spec: . /invoker-scripts/configureDNS.sh && {{- end }} /init.sh --uniqueName $INVOKER_NAME +{{- if .Values.invoker.resources }} + resources: +{{ toYaml .Values.invoker.resources | indent 10 }} +{{- end }} env: - name: "PORT" value: {{ .Values.invoker.port | quote }} diff --git a/helm/openwhisk/templates/kafka-pod.yaml b/helm/openwhisk/templates/kafka-pod.yaml index f49d30f4..a4741f4a 100644 --- a/helm/openwhisk/templates/kafka-pod.yaml +++ b/helm/openwhisk/templates/kafka-pod.yaml @@ -73,6 +73,10 @@ spec: ports: - containerPort: {{ .Values.kafka.port }} name: kafka +{{- if .Values.kafka.resources }} + resources: +{{ toYaml .Values.kafka.resources | indent 10 }} +{{- end }} env: - name: "HOSTNAME_COMMAND" value: "hostname -f" diff --git a/helm/openwhisk/templates/nginx-pod.yaml b/helm/openwhisk/templates/nginx-pod.yaml index ffdf0b12..8e7bb08d 100644 --- a/helm/openwhisk/templates/nginx-pod.yaml +++ b/helm/openwhisk/templates/nginx-pod.yaml @@ -75,6 +75,10 @@ spec: containerPort: {{ .Values.nginx.httpPort }} - name: http-api containerPort: {{ .Values.nginx.httpsPort }} +{{- if .Values.nginx.resources }} + resources: +{{ toYaml .Values.nginx.resources | indent 10 }} +{{- end }} volumeMounts: - name: nginx-conf mountPath: "/etc/nginx/nginx.conf" diff --git a/helm/openwhisk/templates/redis-pod.yaml b/helm/openwhisk/templates/redis-pod.yaml index d5dc44c8..d205f2ec 100644 --- a/helm/openwhisk/templates/redis-pod.yaml +++ b/helm/openwhisk/templates/redis-pod.yaml @@ -83,4 +83,8 @@ spec: ports: - name: redis containerPort: {{ .Values.redis.port }} +{{- if .Values.redis.resources }} + resources: +{{ toYaml .Values.redis.resources | indent 12 }} +{{- end }} {{ end }} diff --git a/helm/openwhisk/templates/zookeeper-pod.yaml b/helm/openwhisk/templates/zookeeper-pod.yaml index 1f01d73a..277162ff 100644 --- a/helm/openwhisk/templates/zookeeper-pod.yaml +++ b/helm/openwhisk/templates/zookeeper-pod.yaml @@ -75,7 +75,10 @@ spec: containerPort: {{ .Values.zookeeper.serverPort }} - name: leader-election containerPort: {{ .Values.zookeeper.leaderElectionPort }} - +{{- if .Values.zookeeper.resources }} + resources: +{{ toYaml .Values.zookeeper.resources | indent 10 }} +{{- end }} livenessProbe: tcpSocket: port: {{ .Values.zookeeper.port }} diff --git a/helm/openwhisk/values.yaml b/helm/openwhisk/values.yaml index 99973206..5b253d9d 100644 --- a/helm/openwhisk/values.yaml +++ b/helm/openwhisk/values.yaml @@ -186,6 +186,7 @@ zookeeper: syncLimit: 2 dataDir: "/data" dataLogDir: "/datalog" + resources: ~ # kafka configurations kafka: @@ -199,6 +200,7 @@ kafka: port: 9092 persistence: size: 512Mi + resources: ~ # Database configuration db: @@ -246,6 +248,7 @@ nginx: cert_file: "" key_file: "" sslPassword: "" + resources: ~ # Controller configurations controller: @@ -259,6 +262,7 @@ controller: jvmHeapMB: "1024" jvmOptions: "" loglevel: "INFO" + resources: ~ # Invoker configurations invoker: @@ -289,6 +293,7 @@ invoker: kubernetes: isolateUserActions: true replicaCount: 1 + resources: ~ # API Gateway configurations apigw: @@ -300,6 +305,7 @@ apigw: restartPolicy: "Always" apiPort: 9000 mgmtPort: 8080 + resources: ~ # Redis (used by apigateway) redis: @@ -314,6 +320,7 @@ redis: port: 6379 persistence: size: 256Mi + resources: ~ # User-events configuration user_events: From c568e234df98456dacd8daaf3638dce80d19fb21 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Fri, 5 Feb 2021 16:13:45 -0500 Subject: [PATCH 03/18] support limited capabilities for controller, invoker, and apigw pods --- helm/openwhisk/templates/apigateway-pod.yaml | 6 ++++++ helm/openwhisk/templates/controller-pod.yaml | 5 +++++ helm/openwhisk/templates/invoker-pod.yaml | 8 ++++++++ helm/openwhisk/values.yaml | 3 +++ 4 files changed, 22 insertions(+) diff --git a/helm/openwhisk/templates/apigateway-pod.yaml b/helm/openwhisk/templates/apigateway-pod.yaml index 62cfd550..32229e18 100644 --- a/helm/openwhisk/templates/apigateway-pod.yaml +++ b/helm/openwhisk/templates/apigateway-pod.yaml @@ -75,3 +75,9 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-whisk.config key: whisk_api_host_url +{{- if .Values.apigw.secure }} + securityContext: + capabilities: + drop: + - all +{{- end }} diff --git a/helm/openwhisk/templates/controller-pod.yaml b/helm/openwhisk/templates/controller-pod.yaml index 0a18496f..713a5fdb 100644 --- a/helm/openwhisk/templates/controller-pod.yaml +++ b/helm/openwhisk/templates/controller-pod.yaml @@ -79,6 +79,11 @@ spec: {{- if .Values.controller.lean }} securityContext: privileged: true +{{- else if .Values.controller.secure }} + securityContext: + capabilities: + drop: + - all {{- end }} command: ["/bin/bash", "-c", "/init.sh `hostname | awk -F '-' '{print $NF}'`"] ports: diff --git a/helm/openwhisk/templates/invoker-pod.yaml b/helm/openwhisk/templates/invoker-pod.yaml index e84b5b25..af6d7139 100644 --- a/helm/openwhisk/templates/invoker-pod.yaml +++ b/helm/openwhisk/templates/invoker-pod.yaml @@ -238,5 +238,13 @@ spec: ports: - name: invoker containerPort: {{ .Values.invoker.port }} +{{- if .Values.invoker.secure }} + securityContext: + capabilities: + drop: + - all + add: + - SYS_ADMIN +{{- end }} {{ include "openwhisk.invoker.volume_mounts" . }} {{- end }} diff --git a/helm/openwhisk/values.yaml b/helm/openwhisk/values.yaml index 5b253d9d..f4bb8dd7 100644 --- a/helm/openwhisk/values.yaml +++ b/helm/openwhisk/values.yaml @@ -263,6 +263,7 @@ controller: jvmOptions: "" loglevel: "INFO" resources: ~ + secure: false # Invoker configurations invoker: @@ -294,6 +295,7 @@ invoker: isolateUserActions: true replicaCount: 1 resources: ~ + secure: false # API Gateway configurations apigw: @@ -306,6 +308,7 @@ apigw: apiPort: 9000 mgmtPort: 8080 resources: ~ + secure: false # Redis (used by apigateway) redis: From 469ac86149a1b831db897b4d61b497e3396d66f6 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Tue, 16 Feb 2021 18:19:00 -0500 Subject: [PATCH 04/18] add seccomp profile for invoker --- helm/openwhisk/templates/invoker-pod.yaml | 6 +- helm/openwhisk/templates/seccomp-cm.yaml | 425 ++++++++++++++++++++++ helm/openwhisk/templates/seccomp-pod.yaml | 37 ++ 3 files changed, 466 insertions(+), 2 deletions(-) create mode 100644 helm/openwhisk/templates/seccomp-cm.yaml create mode 100644 helm/openwhisk/templates/seccomp-pod.yaml diff --git a/helm/openwhisk/templates/invoker-pod.yaml b/helm/openwhisk/templates/invoker-pod.yaml index af6d7139..ae720a26 100644 --- a/helm/openwhisk/templates/invoker-pod.yaml +++ b/helm/openwhisk/templates/invoker-pod.yaml @@ -43,12 +43,14 @@ spec: labels: name: {{ .Release.Name }}-invoker {{ include "openwhisk.label_boilerplate" . | indent 8 }} - - {{- if .Values.metrics.prometheusEnabled }} annotations: + {{- if .Values.metrics.prometheusEnabled }} prometheus.io/scrape: 'true' prometheus.io/port: '{{ .Values.invoker.port }}' {{- end }} + {{- if .Values.invoker.secure }} + seccomp.security.alpha.kubernetes.io/pod: localhost/invoker.json + {{- end }} spec: {{- if eq .Values.invoker.containerFactory.impl "kubernetes" }} diff --git a/helm/openwhisk/templates/seccomp-cm.yaml b/helm/openwhisk/templates/seccomp-cm.yaml new file mode 100644 index 00000000..b1d8dd1a --- /dev/null +++ b/helm/openwhisk/templates/seccomp-cm.yaml @@ -0,0 +1,425 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-seccomp-profile + labels: + name: {{ .Release.Name }}-seccomp-profile +data: + invoker.json: | + { + "defaultAction": "SCMP_ACT_ERRNO", + "archMap": [ + { + "architecture": "SCMP_ARCH_X86_64", + "subArchitectures": [ + "SCMP_ARCH_X86", + "SCMP_ARCH_X32" + ] + }, + { + "architecture": "SCMP_ARCH_AARCH64", + "subArchitectures": [ + "SCMP_ARCH_ARM" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64" + ] + }, + { + "architecture": "SCMP_ARCH_S390X", + "subArchitectures": [ + "SCMP_ARCH_S390" + ] + } + ], + "syscalls": [ + { + "names": [ + "setgid", + "setuid", + "access", + "bind", + "brk", + "capget", + "capset", + "chdir", + "chmod", + "chown", + "chown32", + "clock_getres", + "clock_getres_time64", + "clock_gettime", + "clock_gettime64", + "close", + "connect", + "copy_file_range", + "creat", + "dup", + "dup2", + "dup3", + "epoll_create", + "epoll_create1", + "epoll_ctl", + "epoll_ctl_old", + "epoll_pwait", + "epoll_pwait2", + "epoll_wait", + "epoll_wait_old", + "eventfd", + "eventfd2", + "execve", + "execveat", + "exit", + "exit_group", + "faccessat", + "faccessat2", + "fadvise64", + "fadvise64_64", + "fallocate", + "fanotify_mark", + "fchdir", + "fchmod", + "fchmodat", + "fchown", + "fchown32", + "fchownat", + "fcntl", + "fcntl64", + "fdatasync", + "fgetxattr", + "flistxattr", + "flock", + "fork", + "fremovexattr", + "fsetxattr", + "fstat", + "fstat64", + "fstatat64", + "fstatfs", + "fstatfs64", + "fsync", + "ftruncate", + "ftruncate64", + "futex", + "futex_time64", + "futimesat", + "getcpu", + "getcwd", + "getdents", + "getdents64", + "getegid", + "getegid32", + "geteuid", + "geteuid32", + "getgid", + "getgid32", + "getgroups", + "getgroups32", + "getitimer", + "getpeername", + "getpgid", + "getpgrp", + "getpid", + "getppid", + "getpriority", + "getrandom", + "getresgid", + "getresgid32", + "getresuid", + "getresuid32", + "getrlimit", + "get_robust_list", + "getrusage", + "getsid", + "getsockname", + "getsockopt", + "get_thread_area", + "gettid", + "gettimeofday", + "getuid", + "getuid32", + "getxattr", + "inotify_add_watch", + "inotify_init", + "inotify_init1", + "inotify_rm_watch", + "ioctl", + "ioprio_get", + "ioprio_set", + "ipc", + "kill", + "lgetxattr", + "link", + "linkat", + "listen", + "listxattr", + "llistxattr", + "_llseek", + "lremovexattr", + "lseek", + "lsetxattr", + "lstat", + "lstat64", + "madvise", + "membarrier", + "memfd_create", + "mincore", + "mkdir", + "mkdirat", + "mknod", + "mknodat", + "mlock", + "mlock2", + "mlockall", + "mmap", + "mmap2", + "mprotect", + "mremap", + "msgctl", + "msgget", + "msgrcv", + "msgsnd", + "msync", + "munmap", + "nanosleep", + "newfstatat", + "_newselect", + "open", + "openat", + "openat2", + "pause", + "pidfd_open", + "pidfd_send_signal", + "pipe", + "pipe2", + "poll", + "ppoll", + "ppoll_time64", + "prctl", + "pread64", + "preadv", + "preadv2", + "prlimit64", + "read", + "readahead", + "readlink", + "readlinkat", + "readv", + "recv", + "recvfrom", + "recvmmsg", + "recvmmsg_time64", + "recvmsg", + "remap_file_pages", + "removexattr", + "rename", + "renameat", + "renameat2", + "restart_syscall", + "rmdir", + "rseq", + "rt_sigaction", + "rt_sigpending", + "rt_sigprocmask", + "rt_sigqueueinfo", + "rt_sigreturn", + "rt_sigsuspend", + "rt_sigtimedwait", + "rt_sigtimedwait_time64", + "rt_tgsigqueueinfo", + "sched_getaffinity", + "sched_getattr", + "sched_getparam", + "sched_get_priority_max", + "sched_get_priority_min", + "sched_getscheduler", + "sched_rr_get_interval", + "sched_rr_get_interval_time64", + "sched_setaffinity", + "sched_setattr", + "sched_setparam", + "sched_setscheduler", + "sched_yield", + "seccomp", + "select", + "semctl", + "semget", + "semop", + "semtimedop", + "semtimedop_time64", + "send", + "sendfile", + "sendfile64", + "sendmmsg", + "sendmsg", + "sendto", + "setgroups", + "setgroups32", + "setitimer", + "setpgid", + "setpriority", + "setregid", + "setregid32", + "setresgid", + "setresgid32", + "setrlimit", + "set_robust_list", + "setsid", + "setsockopt", + "set_thread_area", + "set_tid_address", + "setxattr", + "shmat", + "shmctl", + "shmdt", + "shmget", + "sigaltstack", + "signalfd", + "signalfd4", + "sigprocmask", + "sigreturn", + "socket", + "socketcall", + "socketpair", + "splice", + "stat", + "stat64", + "statfs", + "statfs64", + "statx", + "symlink", + "symlinkat", + "sync", + "sync_file_range", + "syncfs", + "sysinfo", + "tee", + "tgkill", + "time", + "times", + "tkill", + "truncate", + "truncate64", + "ugetrlimit", + "umask", + "uname", + "unlink", + "unlinkat", + "utime", + "utimensat", + "utimensat_time64", + "utimes", + "vfork", + "vmsplice", + "wait4", + "waitid", + "waitpid", + "write", + "writev" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "arch_prctl" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "arches": [ + "amd64", + "x32" + ] + }, + "excludes": {} + }, + { + "names": [ + "bpf", + "clone", + "fanotify_init", + "fsconfig", + "fsmount", + "fsopen", + "fspick", + "lookup_dcookie", + "mount", + "move_mount", + "name_to_handle_at", + "open_tree", + "perf_event_open", + "quotactl", + "setdomainname", + "sethostname", + "setns", + "syslog", + "umount", + "umount2", + "unshare" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + }, + "excludes": {} + }, + { + "names": [ + "clone" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 2114060288, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ], + "arches": [ + "s390", + "s390x" + ] + } + } + ] + } diff --git a/helm/openwhisk/templates/seccomp-pod.yaml b/helm/openwhisk/templates/seccomp-pod.yaml new file mode 100644 index 00000000..8a4b6626 --- /dev/null +++ b/helm/openwhisk/templates/seccomp-pod.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ .Release.Name }}-seccomp + labels: + name: {{ .Release.Name }}-seccomp +{{ include "openwhisk.label_boilerplate" . | indent 4 }} +spec: + selector: + matchLabels: + name: {{ .Release.Name }}-seccomp + template: + metadata: + labels: + name: {{ .Release.Name }}-seccomp + spec: + initContainers: + - name: installer + image: alpine:3.10.0 + command: ["/bin/sh", "-c", "cp -r -L /seccomp/*.json /host/seccomp/"] + volumeMounts: + - name: profiles + mountPath: /seccomp + - name: hostseccomp + mountPath: /host/seccomp + readOnly: false + containers: + - name: pause + image: k8s.gcr.io/pause:3.1 + terminationGracePeriodSeconds: 5 + volumes: + - name: hostseccomp + hostPath: + path: /var/lib/kubelet/seccomp + - name: profiles + configMap: + name: {{ .Release.Name }}-seccomp-profile From 2f83c6bcafacd7cf9871d8d8c912dfb58c191915 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Tue, 16 Feb 2021 19:59:52 -0500 Subject: [PATCH 05/18] add controller seccomp profile --- helm/openwhisk/templates/controller-pod.yaml | 5 +- helm/openwhisk/templates/seccomp-cm.yaml | 418 +++++++++++++++++++ 2 files changed, 422 insertions(+), 1 deletion(-) diff --git a/helm/openwhisk/templates/controller-pod.yaml b/helm/openwhisk/templates/controller-pod.yaml index 713a5fdb..27d56e24 100644 --- a/helm/openwhisk/templates/controller-pod.yaml +++ b/helm/openwhisk/templates/controller-pod.yaml @@ -35,11 +35,14 @@ spec: name: {{ .Release.Name }}-controller {{ include "openwhisk.label_boilerplate" . | indent 8 }} - {{- if .Values.metrics.prometheusEnabled }} annotations: + {{- if .Values.metrics.prometheusEnabled }} prometheus.io/scrape: 'true' prometheus.io/port: '{{ .Values.controller.port }}' {{- end }} + {{- if .Values.controller.secure }} + seccomp.security.alpha.kubernetes.io/pod: localhost/controller.json + {{- end }} spec: serviceAccountName: {{ .Release.Name }}-core diff --git a/helm/openwhisk/templates/seccomp-cm.yaml b/helm/openwhisk/templates/seccomp-cm.yaml index b1d8dd1a..9f324003 100644 --- a/helm/openwhisk/templates/seccomp-cm.yaml +++ b/helm/openwhisk/templates/seccomp-cm.yaml @@ -423,3 +423,421 @@ data: } ] } + controller.json: | + { + "defaultAction": "SCMP_ACT_ERRNO", + "archMap": [ + { + "architecture": "SCMP_ARCH_X86_64", + "subArchitectures": [ + "SCMP_ARCH_X86", + "SCMP_ARCH_X32" + ] + }, + { + "architecture": "SCMP_ARCH_AARCH64", + "subArchitectures": [ + "SCMP_ARCH_ARM" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPS64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPS", + "SCMP_ARCH_MIPS64" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64N32" + ] + }, + { + "architecture": "SCMP_ARCH_MIPSEL64N32", + "subArchitectures": [ + "SCMP_ARCH_MIPSEL", + "SCMP_ARCH_MIPSEL64" + ] + }, + { + "architecture": "SCMP_ARCH_S390X", + "subArchitectures": [ + "SCMP_ARCH_S390" + ] + } + ], + "syscalls": [ + { + "names": [ + "setgid", + "setuid", + "access", + "bind", + "brk", + "capget", + "capset", + "chdir", + "chmod", + "chown", + "chown32", + "clock_getres", + "clock_getres_time64", + "clock_gettime", + "clock_gettime64", + "close", + "connect", + "copy_file_range", + "creat", + "dup", + "dup2", + "dup3", + "epoll_create", + "epoll_create1", + "epoll_ctl", + "epoll_ctl_old", + "epoll_pwait", + "epoll_pwait2", + "epoll_wait", + "epoll_wait_old", + "eventfd", + "eventfd2", + "execve", + "execveat", + "exit", + "exit_group", + "faccessat", + "faccessat2", + "fadvise64", + "fadvise64_64", + "fallocate", + "fanotify_mark", + "fchdir", + "fchmod", + "fchmodat", + "fchown", + "fchown32", + "fchownat", + "fcntl", + "fcntl64", + "fdatasync", + "fgetxattr", + "flistxattr", + "flock", + "fork", + "fremovexattr", + "fsetxattr", + "fstat", + "fstat64", + "fstatat64", + "fstatfs", + "fstatfs64", + "fsync", + "ftruncate", + "ftruncate64", + "futex", + "futex_time64", + "futimesat", + "getcpu", + "getcwd", + "getdents", + "getdents64", + "getegid", + "getegid32", + "geteuid", + "geteuid32", + "getgid", + "getgid32", + "getgroups", + "getgroups32", + "getitimer", + "getpeername", + "getpgid", + "getpgrp", + "getpid", + "getppid", + "getpriority", + "getrandom", + "getresgid", + "getresgid32", + "getresuid", + "getresuid32", + "getrlimit", + "get_robust_list", + "getrusage", + "getsid", + "getsockname", + "getsockopt", + "get_thread_area", + "gettid", + "gettimeofday", + "getuid", + "getuid32", + "getxattr", + "inotify_add_watch", + "inotify_init", + "inotify_init1", + "inotify_rm_watch", + "ioctl", + "ioprio_get", + "ioprio_set", + "ipc", + "kill", + "lgetxattr", + "link", + "linkat", + "listen", + "listxattr", + "llistxattr", + "_llseek", + "lremovexattr", + "lseek", + "lsetxattr", + "lstat", + "lstat64", + "madvise", + "membarrier", + "memfd_create", + "mincore", + "mkdir", + "mkdirat", + "mknod", + "mknodat", + "mlock", + "mlock2", + "mlockall", + "mmap", + "mmap2", + "mprotect", + "mremap", + "msgctl", + "msgget", + "msgrcv", + "msgsnd", + "msync", + "munmap", + "nanosleep", + "newfstatat", + "_newselect", + "open", + "openat", + "openat2", + "pause", + "pidfd_open", + "pidfd_send_signal", + "pipe", + "pipe2", + "poll", + "ppoll", + "ppoll_time64", + "prctl", + "pread64", + "preadv", + "preadv2", + "prlimit64", + "read", + "readahead", + "readlink", + "readlinkat", + "readv", + "recv", + "recvfrom", + "recvmmsg", + "recvmmsg_time64", + "recvmsg", + "remap_file_pages", + "removexattr", + "rename", + "renameat", + "renameat2", + "restart_syscall", + "rmdir", + "rseq", + "rt_sigaction", + "rt_sigpending", + "rt_sigprocmask", + "rt_sigqueueinfo", + "rt_sigreturn", + "rt_sigsuspend", + "rt_sigtimedwait", + "rt_sigtimedwait_time64", + "rt_tgsigqueueinfo", + "sched_getaffinity", + "sched_getattr", + "sched_getparam", + "sched_get_priority_max", + "sched_get_priority_min", + "sched_getscheduler", + "sched_rr_get_interval", + "sched_rr_get_interval_time64", + "sched_setaffinity", + "sched_setattr", + "sched_setparam", + "sched_setscheduler", + "sched_yield", + "seccomp", + "select", + "semctl", + "semget", + "semop", + "semtimedop", + "semtimedop_time64", + "send", + "sendfile", + "sendfile64", + "sendmmsg", + "sendmsg", + "sendto", + "setgroups", + "setgroups32", + "setitimer", + "setpgid", + "setpriority", + "setregid", + "setregid32", + "setresgid", + "setresgid32", + "setrlimit", + "set_robust_list", + "setsid", + "setsockopt", + "set_thread_area", + "set_tid_address", + "setxattr", + "shmat", + "shmctl", + "shmdt", + "shmget", + "sigaltstack", + "signalfd", + "signalfd4", + "sigprocmask", + "sigreturn", + "socket", + "socketcall", + "socketpair", + "splice", + "stat", + "stat64", + "statfs", + "statfs64", + "statx", + "symlink", + "symlinkat", + "sync", + "sync_file_range", + "syncfs", + "sysinfo", + "tee", + "tgkill", + "time", + "times", + "tkill", + "truncate", + "truncate64", + "ugetrlimit", + "umask", + "uname", + "unlink", + "unlinkat", + "utime", + "utimensat", + "utimensat_time64", + "utimes", + "vfork", + "vmsplice", + "wait4", + "waitid", + "waitpid", + "write", + "writev" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": {}, + "excludes": {} + }, + { + "names": [ + "arch_prctl" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "arches": [ + "amd64", + "x32" + ] + }, + "excludes": {} + }, + { + "names": [ + "bpf", + "clone", + "fanotify_init", + "fsconfig", + "fsmount", + "fsopen", + "fspick", + "lookup_dcookie", + "mount", + "move_mount", + "name_to_handle_at", + "open_tree", + "perf_event_open", + "quotactl", + "setdomainname", + "sethostname", + "setns", + "syslog", + "umount", + "umount2", + "unshare" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + }, + "excludes": {} + }, + { + "names": [ + "clone" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 0, + "value": 2114060288, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ], + "arches": [ + "s390", + "s390x" + ] + } + } + ] + } From c3a69608c2d3eeb43fc08811d5c7476eee05948d Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Tue, 16 Feb 2021 21:58:43 -0500 Subject: [PATCH 06/18] update controller seccomp profile --- helm/openwhisk/templates/seccomp-cm.yaml | 84 +++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/helm/openwhisk/templates/seccomp-cm.yaml b/helm/openwhisk/templates/seccomp-cm.yaml index 9f324003..69368e22 100644 --- a/helm/openwhisk/templates/seccomp-cm.yaml +++ b/helm/openwhisk/templates/seccomp-cm.yaml @@ -478,9 +478,11 @@ data: "syscalls": [ { "names": [ - "setgid", - "setuid", + "accept", + "accept4", "access", + "adjtimex", + "alarm", "bind", "brk", "capget", @@ -489,11 +491,16 @@ data: "chmod", "chown", "chown32", + "clock_adjtime", + "clock_adjtime64", "clock_getres", "clock_getres_time64", "clock_gettime", "clock_gettime64", + "clock_nanosleep", + "clock_nanosleep_time64", "close", + "close_range", "connect", "copy_file_range", "creat", @@ -591,6 +598,8 @@ data: "ioprio_set", "ipc", "kill", + "lchown", + "lchown32", "lgetxattr", "link", "linkat", @@ -623,6 +632,8 @@ data: "msgrcv", "msgsnd", "msync", + "munlock", + "munlockall", "munmap", "nanosleep", "newfstatat", @@ -643,6 +654,11 @@ data: "preadv", "preadv2", "prlimit64", + "pselect6", + "pselect6_time64", + "pwrite64", + "pwritev", + "pwritev2", "read", "readahead", "readlink", @@ -696,6 +712,12 @@ data: "sendmmsg", "sendmsg", "sendto", + "setfsgid", + "setfsgid32", + "setfsuid", + "setfsuid32", + "setgid", + "setgid32", "setgroups", "setgroups32", "setitimer", @@ -705,17 +727,24 @@ data: "setregid32", "setresgid", "setresgid32", + "setresuid", + "setresuid32", + "setreuid", + "setreuid32", "setrlimit", "set_robust_list", "setsid", "setsockopt", "set_thread_area", "set_tid_address", + "setuid", + "setuid32", "setxattr", "shmat", "shmctl", "shmdt", "shmget", + "shutdown", "sigaltstack", "signalfd", "signalfd4", @@ -739,6 +768,18 @@ data: "tee", "tgkill", "time", + "timer_create", + "timer_delete", + "timer_getoverrun", + "timer_gettime", + "timer_gettime64", + "timer_settime", + "timer_settime64", + "timerfd_create", + "timerfd_gettime", + "timerfd_gettime64", + "timerfd_settime", + "timerfd_settime64", "times", "tkill", "truncate", @@ -838,6 +879,45 @@ data: "s390x" ] } + }, + { + "names": [ + "clone" + ], + "action": "SCMP_ACT_ALLOW", + "args": [ + { + "index": 1, + "value": 2114060288, + "op": "SCMP_CMP_MASKED_EQ" + } + ], + "comment": "s390 parameter ordering for clone is different", + "includes": { + "arches": [ + "s390", + "s390x" + ] + }, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + } + }, + { + "names": [ + "syslog" + ], + "action": "SCMP_ACT_ALLOW", + "args": [], + "comment": "", + "includes": { + "caps": [ + "CAP_SYSLOG" + ] + }, + "excludes": {} } ] } From 02a9af5d6ecaea3e01994cc5135c9bb700991267 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Mon, 17 May 2021 13:05:16 -0400 Subject: [PATCH 07/18] retry git clone in initdb script --- helm/openwhisk/configMapFiles/initCouchDB/initdb.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh index 4f462783..3e172c85 100755 --- a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh +++ b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh @@ -16,7 +16,11 @@ # # Clone OpenWhisk to get the ansible playbooks needed to initialize CouchDB -git clone https://github.com/apache/openwhisk /openwhisk +until git clone https://github.com/apache/openwhisk /openwhisk +do + echo "Can't git clone yet, trying again" + sleep 2 +done pushd /openwhisk git checkout $OW_GIT_TAG_OPENWHISK popd From 061f28a4bfa3191f86a28d4d78ee9cdaae69d1ac Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Mon, 17 May 2021 14:33:01 -0400 Subject: [PATCH 08/18] try making zookeeper port opaque --- helm/openwhisk/templates/zookeeper-pod.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/openwhisk/templates/zookeeper-pod.yaml b/helm/openwhisk/templates/zookeeper-pod.yaml index 277162ff..a6c97062 100644 --- a/helm/openwhisk/templates/zookeeper-pod.yaml +++ b/helm/openwhisk/templates/zookeeper-pod.yaml @@ -24,6 +24,8 @@ metadata: labels: name: {{ .Release.Name }}-zookeeper {{ include "openwhisk.label_boilerplate" . | indent 4 }} + annotations: + config.linkerd.io/opaque-ports: "{{ .Values.zookeeper.port }}" spec: serviceName: {{ .Release.Name }}-zookeeper podManagementPolicy: "Parallel" From 57e1d667492dea2b263c3f5b00720d7cc24fc027 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 19 May 2021 12:49:58 -0400 Subject: [PATCH 09/18] support specifying extra caps on invoker --- helm/openwhisk/templates/invoker-pod.yaml | 3 +++ helm/openwhisk/templates/zookeeper-pod.yaml | 2 -- helm/openwhisk/values.yaml | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/helm/openwhisk/templates/invoker-pod.yaml b/helm/openwhisk/templates/invoker-pod.yaml index ae720a26..e00ae3a3 100644 --- a/helm/openwhisk/templates/invoker-pod.yaml +++ b/helm/openwhisk/templates/invoker-pod.yaml @@ -247,6 +247,9 @@ spec: - all add: - SYS_ADMIN +{{- range .Values.invoker.extraCaps }} + - {{ . }} +{{- end }} {{- end }} {{ include "openwhisk.invoker.volume_mounts" . }} {{- end }} diff --git a/helm/openwhisk/templates/zookeeper-pod.yaml b/helm/openwhisk/templates/zookeeper-pod.yaml index a6c97062..277162ff 100644 --- a/helm/openwhisk/templates/zookeeper-pod.yaml +++ b/helm/openwhisk/templates/zookeeper-pod.yaml @@ -24,8 +24,6 @@ metadata: labels: name: {{ .Release.Name }}-zookeeper {{ include "openwhisk.label_boilerplate" . | indent 4 }} - annotations: - config.linkerd.io/opaque-ports: "{{ .Values.zookeeper.port }}" spec: serviceName: {{ .Release.Name }}-zookeeper podManagementPolicy: "Parallel" diff --git a/helm/openwhisk/values.yaml b/helm/openwhisk/values.yaml index f4bb8dd7..76fdf80a 100644 --- a/helm/openwhisk/values.yaml +++ b/helm/openwhisk/values.yaml @@ -296,6 +296,7 @@ invoker: replicaCount: 1 resources: ~ secure: false + extraCaps: [] # API Gateway configurations apigw: From b004ff7998ed0c29450e70ca49f1160d51138079 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 19 May 2021 14:43:09 -0400 Subject: [PATCH 10/18] retry gencerts commands --- helm/openwhisk/configMapFiles/genCerts/gencerts.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh index b5dbb199..7b3f1054 100755 --- a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh +++ b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh @@ -19,7 +19,15 @@ if kubectl get secret $NGINX_CERT_SECRET; then echo "using existing $NGINX_CERT_SECRET secret" else echo "generating new $NGINX_CERT_SECRET secret" - genssl.sh "*.$WHISK_API_HOST_NAME" server /cert-gen - kubectl create secret tls $NGINX_CERT_SECRET --cert=/cert-gen/openwhisk-server-cert.pem --key=/cert-gen/openwhisk-server-key.pem -fi + until genssl.sh "*.$WHISK_API_HOST_NAME" server /cert-gen + do + echo "Network not ready yet" + sleep 2 + done + until kubectl create secret tls $NGINX_CERT_SECRET --cert=/cert-gen/openwhisk-server-cert.pem --key=/cert-gen/openwhisk-server-key.pem + do + echo "Network not ready yet" + sleep 2 + done +fi From 4fa07c6a75d13a0fab56dee8c17021d7ad2c3a9d Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 19 May 2021 14:50:21 -0400 Subject: [PATCH 11/18] ensure network is ready for install packages job --- helm/openwhisk/configMapFiles/initCouchDB/initdb.sh | 2 +- helm/openwhisk/configMapFiles/installPackages/myTask.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh index 3e172c85..f0812472 100755 --- a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh +++ b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh @@ -18,7 +18,7 @@ # Clone OpenWhisk to get the ansible playbooks needed to initialize CouchDB until git clone https://github.com/apache/openwhisk /openwhisk do - echo "Can't git clone yet, trying again" + echo "Network not ready yet" sleep 2 done pushd /openwhisk diff --git a/helm/openwhisk/configMapFiles/installPackages/myTask.sh b/helm/openwhisk/configMapFiles/installPackages/myTask.sh index bb390434..693fe6eb 100644 --- a/helm/openwhisk/configMapFiles/installPackages/myTask.sh +++ b/helm/openwhisk/configMapFiles/installPackages/myTask.sh @@ -24,7 +24,11 @@ export PROVIDER_DB_URL=$PROVIDER_DB_PROTOCOL://$PROVIDER_DB_USERNAME:$PROVIDER_D ##### # Clone openwhisk repo to get installRouteMgmt.sh and core/routemgmt -git clone https://github.com/apache/openwhisk openwhisk +until git clone https://github.com/apache/openwhisk openwhisk +do + echo "Network not ready yet" + sleep 2 +done pushd openwhisk git checkout $OW_GIT_TAG_OPENWHISK rm -f /openwhisk/ansible/files/auth.guest /openwhisk/ansible/files/auth.whisk.system From c9c85f6bcce3f78f4bbf1579473de474367407ed Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 19 May 2021 15:54:15 -0400 Subject: [PATCH 12/18] try fixing gencerts --- .../configMapFiles/genCerts/gencerts.sh | 21 +++++++++---------- .../configMapFiles/initCouchDB/initdb.sh | 2 +- .../configMapFiles/installPackages/myTask.sh | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh index 7b3f1054..b7ef3380 100755 --- a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh +++ b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh @@ -15,19 +15,18 @@ # limitations under the License. # +until kubectl get nodes +do + echo "Network not ready yet" + sleep 1 +done + if kubectl get secret $NGINX_CERT_SECRET; then echo "using existing $NGINX_CERT_SECRET secret" else echo "generating new $NGINX_CERT_SECRET secret" - until genssl.sh "*.$WHISK_API_HOST_NAME" server /cert-gen - do - echo "Network not ready yet" - sleep 2 - done - - until kubectl create secret tls $NGINX_CERT_SECRET --cert=/cert-gen/openwhisk-server-cert.pem --key=/cert-gen/openwhisk-server-key.pem - do - echo "Network not ready yet" - sleep 2 - done + genssl.sh "*.$WHISK_API_HOST_NAME" server /cert-gen + kubectl create secret tls $NGINX_CERT_SECRET --cert=/cert-gen/openwhisk-server-cert.pem --key=/cert-gen/openwhisk-server-key.pem fi + +exit 0 diff --git a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh index f0812472..5d1d90a1 100755 --- a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh +++ b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh @@ -19,7 +19,7 @@ until git clone https://github.com/apache/openwhisk /openwhisk do echo "Network not ready yet" - sleep 2 + sleep 1 done pushd /openwhisk git checkout $OW_GIT_TAG_OPENWHISK diff --git a/helm/openwhisk/configMapFiles/installPackages/myTask.sh b/helm/openwhisk/configMapFiles/installPackages/myTask.sh index 693fe6eb..b54d591c 100644 --- a/helm/openwhisk/configMapFiles/installPackages/myTask.sh +++ b/helm/openwhisk/configMapFiles/installPackages/myTask.sh @@ -27,7 +27,7 @@ export PROVIDER_DB_URL=$PROVIDER_DB_PROTOCOL://$PROVIDER_DB_USERNAME:$PROVIDER_D until git clone https://github.com/apache/openwhisk openwhisk do echo "Network not ready yet" - sleep 2 + sleep 1 done pushd openwhisk git checkout $OW_GIT_TAG_OPENWHISK From 6f60bf33630889df9b72a42472e2aeddaa09285d Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 19 May 2021 16:23:47 -0400 Subject: [PATCH 13/18] change command --- helm/openwhisk/configMapFiles/genCerts/gencerts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh index b7ef3380..4344808d 100755 --- a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh +++ b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh @@ -15,7 +15,7 @@ # limitations under the License. # -until kubectl get nodes +until kubectl get secret do echo "Network not ready yet" sleep 1 From 196757f3513fe6ef20e3ac7d5b8c7d5dedc93351 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 19 May 2021 16:33:00 -0400 Subject: [PATCH 14/18] remove check from gencerts --- .../configMapFiles/genCerts/gencerts.sh | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh index 4344808d..6fa530c9 100755 --- a/helm/openwhisk/configMapFiles/genCerts/gencerts.sh +++ b/helm/openwhisk/configMapFiles/genCerts/gencerts.sh @@ -15,18 +15,22 @@ # limitations under the License. # -until kubectl get secret -do - echo "Network not ready yet" - sleep 1 -done - if kubectl get secret $NGINX_CERT_SECRET; then echo "using existing $NGINX_CERT_SECRET secret" else echo "generating new $NGINX_CERT_SECRET secret" - genssl.sh "*.$WHISK_API_HOST_NAME" server /cert-gen - kubectl create secret tls $NGINX_CERT_SECRET --cert=/cert-gen/openwhisk-server-cert.pem --key=/cert-gen/openwhisk-server-key.pem + + until genssl.sh "*.$WHISK_API_HOST_NAME" server /cert-gen + do + echo "Network not ready yet" + sleep 1 + done + + until kubectl create secret tls $NGINX_CERT_SECRET --cert=/cert-gen/openwhisk-server-cert.pem --key=/cert-gen/openwhisk-server-key.pem + do + echo "Network not ready yet" + sleep 1 + done fi exit 0 From 5f6dde37910c195cc3f607b6a2f0b4495c1a4707 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 19 May 2021 16:50:18 -0400 Subject: [PATCH 15/18] make initdb compatible with linkerd --- helm/openwhisk/configMapFiles/initCouchDB/initdb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh index 5d1d90a1..99dcae74 100755 --- a/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh +++ b/helm/openwhisk/configMapFiles/initCouchDB/initdb.sh @@ -91,3 +91,4 @@ echo "Creating ow_kube_couchdb_initialized_marker database" curl --silent -X PUT -u "$COUCHDB_USER:$COUCHDB_PASSWORD" $DB_PROTOCOL://$DB_HOST:$DB_PORT/ow_kube_couchdb_initialized_marker || exit 1 echo "successfully initialized CouchDB for OpenWhisk" +exit 0 From 2587b37614d24e2a328ad2705ea2de7c7cad7766 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Wed, 18 Aug 2021 16:02:39 -0400 Subject: [PATCH 16/18] merge master --- .asf.yaml | 46 ++-- .gitignore | 2 - .travis.yml | 10 +- NOTICE.txt | 2 +- README.md | 158 +++++++------- deploy/README.md | 22 ++ deploy/docker-macOS/mycluster.yaml | 34 +++ deploy/docker-windows/mycluster.yaml | 34 +++ deploy/ibm-public/mycluster-iks.yaml | 44 ++++ deploy/ibm-public/mycluster-roks.yaml | 47 +++++ deploy/kind/kind-cluster.yaml | 26 +++ deploy/kind/mycluster.yaml | 37 ++++ {tools/travis => deploy/kind}/start-kind.sh | 24 +-- docs/configurationChoices.md | 11 +- docs/k8s-aws.md | 4 +- docs/k8s-custom-build-cluster-scaleup.md | 2 +- docs/k8s-docker-for-mac.md | 28 ++- docs/k8s-docker-for-windows.md | 52 ++--- docs/k8s-ibm-private.md | 197 ------------------ docs/k8s-ibm-public.md | 94 +++------ docs/k8s-kind.md | 68 +++--- docs/k8s-technical-requirements.md | 4 +- docs/okd-311.md | 79 ------- docs/openshift-4.md | 126 +++++++++++ ...md => openshift-technical-requirements.md} | 8 +- docs/private-docker-registry.md | 2 +- helm/openwhisk/Chart.yaml | 2 +- helm/openwhisk/README.md | 4 +- .../tests/packageChecker/myTask.sh | 1 - .../configMapFiles/tests/smoketest/myTask.sh | 12 +- helm/openwhisk/runtimes-minimal-travis.json | 8 +- helm/openwhisk/runtimes.json | 54 +++-- helm/openwhisk/templates/NOTES.txt | 2 +- helm/openwhisk/templates/_affinity.tpl | 2 +- helm/openwhisk/templates/_helpers.tpl | 6 +- helm/openwhisk/templates/_readiness.tpl | 5 +- helm/openwhisk/templates/apigateway-pod.yaml | 4 +- helm/openwhisk/templates/controller-pod.yaml | 2 +- helm/openwhisk/templates/couchdb-pod.yaml | 4 + .../templates/frontdoor-ingress.yaml | 46 ++-- .../openwhisk/templates/frontdoor-routes.yaml | 114 ---------- .../templates/frontdoor-secrets.yaml | 4 +- helm/openwhisk/templates/gen-certs-cm.yaml | 2 + helm/openwhisk/templates/gen-certs-job.yaml | 4 +- .../templates/install-packages-job.yaml | 8 +- helm/openwhisk/templates/kafka-pod.yaml | 15 ++ helm/openwhisk/templates/nginx-cm.yaml | 6 +- helm/openwhisk/templates/nginx-pod.yaml | 8 +- helm/openwhisk/templates/nginx-svc.yaml | 7 +- helm/openwhisk/templates/ow-whisk-cm.yaml | 35 ++-- .../templates/provider-alarm-pod.yaml | 8 +- .../templates/provider-kafka-pod.yaml | 4 +- helm/openwhisk/templates/redis-pod.yaml | 4 + .../templates/tests/package-checker-pod.yaml | 2 +- .../templates/tests/smoketest-pod.yaml | 2 +- .../templates/tests/systemtest-pod.yaml | 2 +- helm/openwhisk/values.schema.json | 9 +- helm/openwhisk/values.yaml | 16 +- tools/travis/deploy-chart.sh | 36 +--- tools/travis/setup.sh | 2 +- 60 files changed, 800 insertions(+), 801 deletions(-) create mode 100644 deploy/README.md create mode 100644 deploy/docker-macOS/mycluster.yaml create mode 100644 deploy/docker-windows/mycluster.yaml create mode 100644 deploy/ibm-public/mycluster-iks.yaml create mode 100644 deploy/ibm-public/mycluster-roks.yaml create mode 100644 deploy/kind/kind-cluster.yaml create mode 100644 deploy/kind/mycluster.yaml rename {tools/travis => deploy/kind}/start-kind.sh (50%) delete mode 100644 docs/k8s-ibm-private.md delete mode 100644 docs/okd-311.md create mode 100644 docs/openshift-4.md rename docs/{okd-technical-requirements.md => openshift-technical-requirements.md} (84%) delete mode 100644 helm/openwhisk/templates/frontdoor-routes.yaml diff --git a/.asf.yaml b/.asf.yaml index 6865ecb9..48634c37 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -19,15 +19,37 @@ github: description: "The Apache OpenWhisk Kubernetes Deployment repository supports deploying the Apache OpenWhisk system on Kubernetes and OpenShift clusters." homepage: https://openwhisk.apache.org/ labels: - - openwhisk - - apache - - serverless - - faas - - functions-as-a-service - - cloud - - serverless-architectures - - serverless-functions - - docker - - kubernetes - - functions - - openshift + - openwhisk + - apache + - serverless + - faas + - functions-as-a-service + - cloud + - serverless-architectures + - serverless-functions + - docker + - kubernetes + - functions + - openshift + protected_branches: + master: + required_status_checks: + strict: false + contexts: + - "Travis CI - Pull Request" + required_pull_request_reviews: + required_appoving_review_count: 1 + required_signatures: false + enabled_merge_buttons: + merge: false + squash: true + rebase: true + features: + issues: true + +notifications: + commits: commits@openwhisk.apache.org + issues_status: issues@openwhisk.apache.org + issues_comment: issues@openwhisk.apache.org + pullrequests_status: issues@openwhisk.apache.org + pullrequests_comment: issues@openwhisk.apache.org diff --git a/.gitignore b/.gitignore index 097f99e0..6491baa4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ kubernetes/nginx/certs -helm/mycluster.yaml -mycluster.yaml myruntimes.json .DS_Store *~ diff --git a/.travis.yml b/.travis.yml index 00816000..91cd820c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,10 +23,10 @@ env: - secure: d7CuMXbhT83W2x78qiLwgogX1+3aPicd1PlTwwNNDN6QSkImbxareyKThnsqlHIiNj3o5l5DBuiYjy7wrF/xD1g8BQMmTwm99DRx5q3CI3Im3VCi/ZK8SaNjuOy24d7cf5k2tB/87Gk7zmKsMDYm+fpCl+GpgUmIEeIwthiAxuXSDWZ8eQPIptmxj56DeFRNouvXG+dEUtBfWiwN27UPxNKExCixFnegmdtffLbz6hhst7BHr5Ry9acbycre98PCwWZcu9lxFs+SJ1kvnzX2iue4otmDkF1WkJjxaOFPJVs/D3YItg+neLCSxjwBskPed+Fct8bOjcM/uVROJPNIq5icBmaPX2isH0lvtxOeVw/dmioWYXYPN9ygBOe4eO/vtPllN0bcAUo5xl9jXev8ciAozYrYpHVh9Fplfd81rcYTeYzALmRJBdoiWoc3KQGzwGc9sB1ffmy+KWgG9T0zbnS4fALSR4PSzyNlKSLXw9vuvdNP0OBYtO+6yTJXavIxqmDoj64Lb93n+uGPatnaIGPTKEEBMJTSjsgYVEfxzzZuxUT9Ldkf2lzqvN2PCllGoMqxkgsdb8i4v4QgRaMWBDbKa5Va4k0O4dnhRmtdbJavOSKN6fECJbyfoJlV1VvJGxk5znVLRznBmUPBKbNccyPZJULugKD3QIh4q8Q5jBU= - secure: CJtnU94HTDqd4A6uvhFl8IpnmU+wTdlzb8bPBFUl/lI/VKXiRrYpgJdKUro5xEoxFKuqMprLhbyf66niyWLTIeogjUAEu/h/o2dBVeGgSGKoqC0hQgqvnxKFeGlzFJ0XuEs3vbStJGRnQszGsfnnDrscJtR0x9X+1w4aBKI7iPyyuFtVkDD1UsmBbSi+M8FTeq7G7A0reMDaey7uog3CFCpIMl4geshcohQEcKEGbnXQZoLPFpb7cBOE83VXBJ7Y7Dgf/U4keiLovvnuJThGKZm/SVV2KlELmBmtmbx3rMT6Vb5k9ChSdRWapromNnnzmJBIQ5Scc2mwV3A93/SMha1F3IlYpDKs5djfTw8jZfVnuiou7HhTaRjHkmmcwP12/k30gLe2kw0Vezg1TCY4zgtOpcmCxc8RHEy0ceA74rKvRi8LbexTCwX+iAMQFn/pSrh/OqAq/50JbLyczcoO1zXWS38txUQNLW8i+XllhCg9pjkjyfBeGjOOcWiVIz9rWJd2XufjSXDcj6xoZHtkh1XDt1CnVkpsYKtyyZucQnhUM9ebmaWqbSW2+bpqC/2hI+G+kOyyCesGdB1q+VmN1augMMs6RgWjk4yw5dyLAshATSoUlE8KH2cDcJL19r4ECaQ99PSLwxoB89yfPoJiNc42vwxRdsLmB1BMNyPa81Y= matrix: - - TRAVIS_KUBE_VERSION=v1.17 OW_INCLUDE_SYSTEM_TESTS=true OW_CONTAINER_FACTORY=kubernetes - - TRAVIS_KUBE_VERSION=v1.18 OW_INCLUDE_SYSTEM_TESTS=true OW_CONTAINER_FACTORY=kubernetes - - TRAVIS_KUBE_VERSION=v1.18 OW_INCLUDE_SYSTEM_TESTS=false OW_CONTAINER_FACTORY=kubernetes OW_LEAN_MODE=true - - TRAVIS_KUBE_VERSION=v1.19 OW_INCLUDE_SYSTEM_TESTS=false OW_CONTAINER_FACTORY=kubernetes + - TRAVIS_KUBE_VERSION=v1.19 OW_INCLUDE_SYSTEM_TESTS=true OW_CONTAINER_FACTORY=kubernetes + - TRAVIS_KUBE_VERSION=v1.19 OW_INCLUDE_SYSTEM_TESTS=false OW_CONTAINER_FACTORY=kubernetes OW_LEAN_MODE=true + - TRAVIS_KUBE_VERSION=v1.20 OW_INCLUDE_SYSTEM_TESTS=false OW_CONTAINER_FACTORY=kubernetes + - TRAVIS_KUBE_VERSION=v1.21 OW_INCLUDE_SYSTEM_TESTS=false OW_CONTAINER_FACTORY=kubernetes services: - docker @@ -42,7 +42,7 @@ before_install: - ./tools/travis/setupscan.sh - ./tools/travis/scancode.sh - ./tools/travis/setup.sh - - ./tools/travis/start-kind.sh + - ./deploy/kind/start-kind.sh script: - ./tools/travis/deploy-chart.sh diff --git a/NOTICE.txt b/NOTICE.txt index c5c7b843..25652f24 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Apache OpenWhisk Deploy Kubernetes -Copyright 2016-2020 The Apache Software Foundation +Copyright 2016-2021 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 3ac083b9..a4771453 100644 --- a/README.md +++ b/README.md @@ -31,17 +31,10 @@ language, that can be dynamically scheduled and run in response to associated events (via Triggers) from external sources (Feeds) or from HTTP requests. -This repository supports deploying OpenWhisk to Kubernetes. +This repository supports deploying OpenWhisk to Kubernetes and OpenShift. It contains a Helm chart that can be used to deploy the core OpenWhisk platform and optionally some of its Event Providers -to both single-node and multi-node Kubernetes clusters. - -The same Helm chart can also be used to deploy OpenWhisk to -OKD/OpenShift via a strategy of using `helm template` to -generate yaml that is then fed to the `oc` cli. There are some -rough edges still in this process, we would welcome community -contributions to help improve the targeting of OKD/OpenShift and -document the necessary steps. +to both single-node and multi-node Kubernetes and OpenShift clusters. # Table of Contents @@ -101,29 +94,28 @@ by scaling up the replica count of the various components and labeling multiple nodes as invoker nodes. We have detailed documentation on using Kubernetes clusters from the following major cloud providers: -* [IBM (IKS)](docs/k8s-ibm-public.md) and [IBM (ICP)](docs/k8s-ibm-private.md) +* [IBM (IKS)](docs/k8s-ibm-public.md) * [Google (GKE)](docs/k8s-google.md) * [Amazon (EKS)](docs/k8s-aws.md) We would welcome contributions of documentation for Azure (AKS) and any other public cloud providers. -### Using OKD/OpenShift +### Using OpenShift You will need at least 1 worker node with 4GB of memory and 2 virtual CPUs to deploy the default configuration of OpenWhisk. You can deploy to significantly larger clusters by scaling up the replica count of the various components and labeling multiple nodes as invoker nodes. For more detailed documentation, see: -* [OKD/OpenShift 3.11](docs/okd-311.md) +* [OpenShift 4](docs/openshift-4.md) ### Using a Kubernetes cluster you built yourself If you are comfortable with building your own Kubernetes clusters and deploying services with ingresses to them, you should also be able to deploy OpenWhisk to a do-it-yourself cluster. Make sure -your cluster meets the [technical -requirements](docs/k8s-technical-requirements.md). You will need at -least 1 worker node with 4GB of memory and 2 virtual CPUs to deploy +your cluster meets the [technical requirements](docs/k8s-technical-requirements.md). +You will need at least 1 worker node with 4GB of memory and 2 virtual CPUs to deploy the default configuration of OpenWhisk. You can deploy to significantly larger clusters by scaling up the replica count of the various components and labeling multiple nodes as invoker nodes. @@ -136,7 +128,7 @@ Additional more detailed instructions: [Helm](https://github.com/kubernetes/helm) is a tool to simplify the deployment and management of applications on Kubernetes clusters. -The OpenWhisk Helm chart requires the Helm 3. +The OpenWhisk Helm chart requires Helm 3. Our automated testing currently uses Helm v3.2.4 @@ -153,12 +145,15 @@ the Helm 3 CLI, you are ready to deploy OpenWhisk. You will use Helm to deploy OpenWhisk to your Kubernetes cluster. There are four deployment steps that are described in more detail below in the rest of this section. -1. [Initial cluster setup](#initial-setup). You will label your -Kubernetes worker nodes to indicate their intended usage by OpenWhisk. +1. [Initial cluster setup](#initial-setup). If you have provisioned a +multi-node cluster, you should label the worker nodes +to indicate their intended usage by OpenWhisk. 2. [Customize the deployment](#customize-the-deployment). You will create a `mycluster.yaml` that specifies key facts about your Kubernetes cluster and the OpenWhisk configuration you wish to -deploy. +deploy. Predefined `mycluster.yaml` files for common flavors +of Kubernetes clusters are provided in the [deploy](./deploy) +directory. 3. [Deploy OpenWhisk with Helm](#deploy-with-helm). You will use Helm and `mycluster.yaml` to deploy OpenWhisk to your Kubernetes cluster. 4. [Configure the `wsk` CLI](#configure-the-wsk-cli). You need to @@ -166,26 +161,42 @@ tell the `wsk` CLI how to connect to your OpenWhisk deployment. ## Initial setup -Indicate the Kubernetes worker nodes that should be used to execute -user containers by OpenWhisk's invokers. Do this by labeling each node with +### Single Worker Node Clusters + +If your cluster has a single worker node, then you should +configure OpenWhisk without node affinity. This is done by adding +the following lines to your `mycluster.yaml` +``` +affinity: + enabled: false + +toleration: + enabled: false + +invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" +``` + +### Multi Worker Node Clusters + +If you are deploying OpenWhisk to a cluster with multiple worker +nodes, we recommend using node affinity to segregate the compute nodes +used for the OpenWhisk control plane from those used to execute user +functions. Do this by labeling each node with `openwhisk-role=invoker`. In the default configuration, which uses the KubernetesContainerFactory, the node labels are used in conjunction with Pod affinities to inform the Kubernetes scheduler how to place work so that user actions will not interfere with the OpenWhisk control plane. When using the non-default DockerContainerFactory, -OpenWhisk assumes it has exclusive use of these invoker nodes and -will schedule work on them directly, completely bypassing the Kubernetes -scheduler. For a single node cluster, simply do -```shell -kubectl label nodes --all openwhisk-role=invoker -``` -If you have a multi-node cluster, then for each node -you want to be an invoker, execute +OpenWhisk assumes it has exclusive use of these invoker nodes and will +schedule work on them directly, completely bypassing the Kubernetes +scheduler. For each node + you want to be an invoker, execute ```shell -$ kubectl label nodes openwhisk-role=invoker +kubectl label node openwhisk-role=invoker ``` -If you are targeting OKD/OpenShift, use the command +If you are targeting OpenShift, use the command ```shell oc label node openwhisk-role=invoker ``` @@ -200,36 +211,30 @@ If you have dedicated Ingress nodes, label them with Event Providers on specific nodes, label those nodes with `openwhisk-role=provider`. -### Disabling affinity (not recommended) -If the Kubernetes cluster does not allow you to assign a label to a node, or you cannot use the affinity attribute, you can disable it. -Please note that it is suitable for testing purposes only and may interfere with the OpenWhisk control plane. - -You can disable affinity by editing the `mycluster.yaml` file: - -```yaml -affinity: - enabled: false -invoker: - options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" -``` +If the Kubernetes cluster does not allow you to assign a label to a +node, or you cannot use the affinity attribute, you use the yaml +snippet shown above in the single worker node configuration to disable +the use of affinities by OpenWhisk. ## Customize the Deployment -You must create a `mycluster.yaml` file to record key aspects of your +You will need a `mycluster.yaml` file to record key aspects of your Kubernetes cluster that are needed to configure the deployment of OpenWhisk to your cluster. For details, see the documentation appropriate to your Kubernetes cluster: * [Docker for Mac](docs/k8s-docker-for-mac.md#configuring-openwhisk) * [Docker for Windows](docs/k8s-docker-for-windows.md#configuring-openwhisk) * [kind](docs/k8s-kind.md#configuring-openwhisk) -* [IBM Kubernetes Service (IKS)](docs/k8s-ibm-public.md#configuring-openwhisk) -* [IBM Cloud Private (ICP)](docs/k8s-ibm-private.md#configuring-openwhisk) +* [IBM Cloud Kubernetes Service (IKS)](docs/k8s-ibm-public.md#configuring-openwhisk) * [Google (GKE)](docs/k8s-google.md#configuring-openwhisk) * [Amazon (EKS)](docs/k8s-aws.md#configuring-openwhisk) -* [OKD/OpenShift](docs/okd-311.md##configuring-openwhisk) +* [OpenShift](docs/openshift-4.md##configuring-openwhisk) + +Default/template `mycluster.yaml` for various types of Kubernetes clusets +can be found in subdirectories of [deploy](./deploy). -Beyond the Kubernetes cluster specific configuration information, -the `mycluster.yaml` file is also used +Beyond the basic Kubernetes cluster specific configuration information, +the `mycluster.yaml` file can also be used to customize your OpenWhisk deployment by enabling optional features and controlling the replication factor of the various microservices that make up the OpenWhisk implementation. See the [configuration @@ -243,7 +248,7 @@ For simplicity, in this README, we have used `owdev` as the release name and You can use a different name and/or namespace simply by changing the commands used below. -**NOTE:** The commands below assume Helm v3.2.0 or higher. Verfiy your local Helm version with the command `helm version`. +**NOTE:** The commands below assume Helm v3.2.0 or higher. Verify your local Helm version with the command `helm version`. ### Deploying Released Charts from Helm Repository @@ -261,24 +266,12 @@ helm install owdev openwhisk/openwhisk -n openwhisk --create-namespace -f myclus To deploy directly from sources, either download the [latest source release](https://github.com/apache/openwhisk-deploy-kube/releases) or `git clone https://github.com/apache/openwhisk-deploy-kube.git` and use the Helm chart -from the `helm/openwhisk` folder of the sourc tree. +from the `helm/openwhisk` folder of the source tree. ```shell helm install owdev ./helm/openwhisk -n openwhisk --create-namespace -f mycluster.yaml ``` -### Deploying to OKD/OpenShift - -Deploying to OKD/OpenShift currently requires a two command sequence: -```shell -helm template owdev ./helm/openwhisk -n openwhisk -f mycluster.yaml > owdev.yaml -oc create -f owdev.yaml -``` -The two step sequence is currently required because the `oc` command must be -used to create the `Route` resource specified in the generated `owdev.yaml`. -We recommend generating to a file to make it easier to undeploy OpenWhisk later -by simply doing `oc delete -f owdev.yaml` - ### Checking status You can use the command `helm status owdev -n openwhisk` to get a summary @@ -294,7 +287,7 @@ Configure the OpenWhisk CLI, wsk, by setting the auth and apihost properties (if you don't already have the wsk cli, follow the instructions [here](https://github.com/apache/openwhisk-cli) to get it). Replace `whisk.ingress.apiHostName` and `whisk.ingress.apiHostPort` -with the actual values from your mycluster.yaml. +with the actual values from your `mycluster.yaml`. ```shell wsk property set --apihost : wsk property set --auth 23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP @@ -306,7 +299,7 @@ host environment. Instead, exposed NodePorts are forwarded from localhost to the appropriate containers. This means that you will use `localhost` instead of `whisk.ingress.apiHostName` when configuring the `wsk` cli and replace `whisk.ingress.apiHostPort` -with the actual values from your mycluster.yaml. +with the actual values from your `mycluster.yaml`. ```shell wsk property set --apihost localhost: @@ -334,7 +327,7 @@ If your deployment is not working, check our Using defaults, your deployment is configured to provide a bare-minimum working platform for testing and exploration. For your specialized workloads, you can scale-up your openwhisk deployment by defining your deployment configurations in your `mycluster.yaml` which overrides the defaults in `helm/openwhisk/values.yaml`. Some important parameters to consider (for other parameters, check `helm/openwhisk/values.yaml` and [configurationChoices](./docs/configurationChoices.md)): * `actionsInvokesPerminute`: limits the maximum number of invocations per minute. -* `actionsInvokesPerminute`: limits the maximum concurrent invocations. +* `actionsInvokesConcurrent`: limits the maximum concurrent invocations. * `containerPool`: total memory available per `invoker` instance. `Invoker` uses this memory to create containers for user-actions. The concurrency-limit (actions running in parallel) will depend upon the total memory configured for `containerPool` and memory allocated per action (`default:` 256mb per container). For more information about increasing concurrency-limit, check [scaling-up your deployment](./docs/k8s-custom-build-cluster-scaleup.md). @@ -409,22 +402,42 @@ controller: ### Selectively redeploying using a locally built docker image You can use the `helm upgrade` command to selectively redeploy one or -more OpenWhisk componenets. Continuing the example above, if you make +more OpenWhisk components. Continuing the example above, if you make additional changes to the controller source code and want to just redeploy it without redeploying the entire OpenWhisk system you can do the following: + +If you are using a multi-node Kubernetes cluster you will need to +repeat the following steps on all nodes that may run the controller +component. + +The first step is to rebuild the docker image: ```shell -# Execute these commands in your openwhisk directory +# Execute this command in your openwhisk directory +bin/wskdev controller -b +``` +Note that the ```wskdev``` flags ```-x``` and ```-d``` are not compatible +with the Kubernetes deployment of OpenWhisk. + +Alternatively, you can build all of the OpenWhisk docker components: +```shell +# Execute this command in your openwhisk directory ./gradlew distDocker +``` + +After building the new docker image(s), tag the new image: +```shell +# Tag the docker image you seek to redeploy docker tag whisk/controller whisk/controller:v2 ``` + Then, edit your `mycluster.yaml` to contain: ```yaml controller: imageName: "whisk/controller" imageTag: "v2" ``` -Redeploy with Helm by executing this commaned in your +Redeploy with Helm by executing this command in your openwhisk-deploy-kube directory: ```shell helm upgrade owdev ./helm/openwhisk -n openwhisk -f mycluster.yaml @@ -447,13 +460,6 @@ helm uninstall owdev -n openwhisk By default, `helm uninstall` removes the history of previous deployments. If you want to keep the history, add the command line flag `--keep-history`. -For OpenShift deployments, you cannot use `helm uninstall` to remove the OpenWhisk -deployment because we did not do a `helm install`. -If you saved the output from `helm template` into a file, -you can simply use that file as an argument to `oc delete`. If you -did not save the file, you can redo the `helm template` command and -feed the generated yaml into an `oc delete` command. - # Issues If your OpenWhisk deployment is not working, check our diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..35c02adf --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,22 @@ + + +This directory contains scripts and configuration files for configuring +various flavors of Kubernetes cluster providers. + diff --git a/deploy/docker-macOS/mycluster.yaml b/deploy/docker-macOS/mycluster.yaml new file mode 100644 index 00000000..96098a97 --- /dev/null +++ b/deploy/docker-macOS/mycluster.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +whisk: + ingress: + type: NodePort + apiHostName: localhost + apiHostPort: 31001 + useInternally: false + +nginx: + httpsNodePort: 31001 + +# A single node cluster; so disable affinity +affinity: + enabled: false +toleration: + enabled: false +invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" diff --git a/deploy/docker-windows/mycluster.yaml b/deploy/docker-windows/mycluster.yaml new file mode 100644 index 00000000..96098a97 --- /dev/null +++ b/deploy/docker-windows/mycluster.yaml @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +whisk: + ingress: + type: NodePort + apiHostName: localhost + apiHostPort: 31001 + useInternally: false + +nginx: + httpsNodePort: 31001 + +# A single node cluster; so disable affinity +affinity: + enabled: false +toleration: + enabled: false +invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" diff --git a/deploy/ibm-public/mycluster-iks.yaml b/deploy/ibm-public/mycluster-iks.yaml new file mode 100644 index 00000000..82bed124 --- /dev/null +++ b/deploy/ibm-public/mycluster-iks.yaml @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +whisk: + ingress: + # NOTE: Replace with your cluster's actual domain + apiHostName: openwhisk. + apiHostPort: 443 + apiHostProto: https + type: Standard + useInternally: true + # NOTE: Replace with your cluster's actual domain + domain: openwhisk. + tls: + enabled: true + secretenabled: true + createsecret: false + # NOTE: Replace with your cluster's actual tlssecret + secretname: + annotations: + kubernetes.io/ingress.class: public-iks-k8s-nginx + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_set_header X-Request-ID $request_id; + nginx.ingress.kubernetes.io/proxy-body-size: 50m + nginx.ingress.kubernetes.io/proxy-read-timeout: "75" + +invoker: + containerFactory: + impl: kubernetes diff --git a/deploy/ibm-public/mycluster-roks.yaml b/deploy/ibm-public/mycluster-roks.yaml new file mode 100644 index 00000000..f5b2c2bc --- /dev/null +++ b/deploy/ibm-public/mycluster-roks.yaml @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +whisk: + ingress: + # NOTE: Replace with your cluster's actual domain + apiHostName: openwhisk. + apiHostPort: 443 + apiHostProto: https + type: Standard + useInternally: true + # NOTE: Replace with your cluster's actual domain + domain: openwhisk. + tls: + enabled: true + secretenabled: true + createsecret: false + # NOTE: Replace with your cluster's actual tlssecret + secretname: + annotations: + kubernetes.io/ingress.class: public-iks-k8s-nginx + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_set_header X-Request-ID $request_id; + nginx.ingress.kubernetes.io/proxy-body-size: 50m + nginx.ingress.kubernetes.io/proxy-read-timeout: "75" + +k8s: + dns: dns-default.openshift-dns + +invoker: + containerFactory: + impl: kubernetes diff --git a/deploy/kind/kind-cluster.yaml b/deploy/kind/kind-cluster.yaml new file mode 100644 index 00000000..b9dc1f89 --- /dev/null +++ b/deploy/kind/kind-cluster.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker + extraPortMappings: + - hostPort: 31001 + containerPort: 31001 +- role: worker diff --git a/deploy/kind/mycluster.yaml b/deploy/kind/mycluster.yaml new file mode 100644 index 00000000..c4e56435 --- /dev/null +++ b/deploy/kind/mycluster.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +whisk: + ingress: + type: NodePort + apiHostName: localhost + apiHostPort: 31001 + useInternally: false + +nginx: + httpsNodePort: 31001 + +# disable affinity +affinity: + enabled: false +toleration: + enabled: false +invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" + # must use KCF as kind uses containerd as its container runtime + containerFactory: + impl: "kubernetes" diff --git a/tools/travis/start-kind.sh b/deploy/kind/start-kind.sh similarity index 50% rename from tools/travis/start-kind.sh rename to deploy/kind/start-kind.sh index 2a0cd7ae..9f902faa 100755 --- a/tools/travis/start-kind.sh +++ b/deploy/kind/start-kind.sh @@ -18,30 +18,20 @@ set -x -# Create cluster config -cat > mycluster.yaml < ``` + If has a dynamic provisioner, deploying the Helm chart will automatically create the required PersistentVolumes. If does not have a dynamic provisioner, then you will need to manually create the required persistent volumes. Alternatively, you may also entirely disable the usage of persistence -by adding the following stanza to your mycluster.yaml: +by adding the following stanza to your `mycluster.yaml`: + ```yaml k8s: persistence: diff --git a/docs/k8s-aws.md b/docs/k8s-aws.md index b73e8681..4b36a215 100644 --- a/docs/k8s-aws.md +++ b/docs/k8s-aws.md @@ -64,8 +64,10 @@ A typical output would be as shown below ] } ``` -Add the following to your mycluster.yaml, using your certificate's Arn + +Add the following to your `mycluster.yaml`, using your certificate's Arn instead of the example one: + ```yaml whisk: ingress: diff --git a/docs/k8s-custom-build-cluster-scaleup.md b/docs/k8s-custom-build-cluster-scaleup.md index e2903715..36327949 100644 --- a/docs/k8s-custom-build-cluster-scaleup.md +++ b/docs/k8s-custom-build-cluster-scaleup.md @@ -29,7 +29,7 @@ The default configurations of openwhisk deployment, support low concurrency-limi By default, openwhisk deployment is configured to provide a bare-minimum working platform for testing and exploration. For your specialized workloads, you can scale-up your openwhisk deployment by defining your deployment configurations in your `mycluster.yaml` which overrides the defaults in `helm/openwhisk/values.yaml`. Some important parameters to consider (for other parameters, check `helm/openwhisk/values.yaml` and [configurationChoices](./docs/configurationChoices.md)): * `actionsInvokesPerminute`: limits the maximum number of invocations per minute. -* `actionsInvokesPerminute`: limits the maximum concurrent invocations. +* `actionsInvokesConcurrent`: limits the maximum concurrent invocations. * `containerPool.userMemory`: total memory available per `invoker` instance. `Invoker` uses this memory to create containers for user-actions. The concurrency-limit (actions running in parallel) will depend upon the total memory configured for `containerPool` and memory allocated per action (`default:` 256mb per container). * `triggersFiresPerminute`: limits the maximum triggers invoked per minute. diff --git a/docs/k8s-docker-for-mac.md b/docs/k8s-docker-for-mac.md index b96b57c0..8cd748ee 100644 --- a/docs/k8s-docker-for-mac.md +++ b/docs/k8s-docker-for-mac.md @@ -50,18 +50,27 @@ might also have installed on your machine. Finally, pick the ### Configuring OpenWhisk You will be using a NodePort ingress to access OpenWhisk. Assuming -`kubectl describe nodes | grep InternalIP` returns 192.168.65.3 and port 31001 is available to be used on your host machine, a -mycluster.yaml for a standard deployment of OpenWhisk would be: +[mycluster.yaml](../deploy/docker-macOS/mycluster.yaml] +for a standard deployment of OpenWhisk would be: ```yaml whisk: ingress: type: NodePort - apiHostName: 192.168.65.3 + apiHostName: localhost apiHostPort: 31001 + useInternally: false nginx: httpsNodePort: 31001 + +# A single node cluster; so disable affinity +affinity: + enabled: false +toleration: + enabled: false +invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" ``` ## Hints and Tips @@ -88,16 +97,3 @@ deployments of OpenWhisk. TLS termination will be handled by OpenWhisk's `nginx` service and will use self-signed certificates. You will need to invoke `wsk` with the `-i` command line argument to bypass certificate checking. - -The docker network is not exposed to the host on MacOS. However, the -exposed ports for NodePort services are forwarded from localhost. -Therefore you must use different host names to connect to OpenWhisk -from outside the cluster (with the `wsk` cli) and from inside the -cluster (in `mycluster.yaml`). Continuing the example from above, -when setting the `--apihost` for the `wsk` cli, you would use -`localhost:31001`. This networking difference also shows up when -listing apis via `wsk -i api list`. The listed URLs will show the -cluster-internal apihost, -e.g. `https://192.168.65.3:31001/api//`, to invoke the api -from outside the cluster you should use `localhost:31001` instead, e.g. -`https://localhost:31001/api//`. diff --git a/docs/k8s-docker-for-windows.md b/docs/k8s-docker-for-windows.md index 8296990c..c9793d9d 100644 --- a/docs/k8s-docker-for-windows.md +++ b/docs/k8s-docker-for-windows.md @@ -53,60 +53,40 @@ In a nutshell, open the Docker preferences window, switch to the allocated to Docker**. Then switch to the Kubernetes panel, and check the box to enable Kubernetes. -### Using Git to Clone this Repository - -`git clone https://github.com/apache/openwhisk-deploy-kube.git` - ### Configuring OpenWhisk You will be using a NodePort ingress to access OpenWhisk. Assuming `kubectl describe nodes | find "InternalIP"` returns 192.168.65.3 and port 31001 is available to be used on your host machine, a -mycluster.yaml for a standard deployment of OpenWhisk would be: +[mycluster.yaml](../deploy/docker-windows/mycluster.yaml) for a standard deployment of OpenWhisk would be: ```yaml whisk: ingress: type: NodePort - apiHostName: 192.168.65.3 + apiHostName: localhost apiHostPort: 31001 + useInternally: false nginx: httpsNodePort: 31001 -``` - -### Using helm to install OpenWhisk - -Indicate the Kubernetes worker nodes that should be used to execute user -containers by OpenWhisk's invokers. For a single node development cluster, -simply run: - -`kubectl label nodes --all openwhisk-role=invoker` -Make sure you created your -`mycluster.yaml` file as described above, and run: - -```cmd -cd openwhisk-deploy-kube -helm install owdev ./helm/openwhisk -n openwhisk --create-namespace -f mycluster.yaml +# A single node cluster; so disable affinity +affinity: + enabled: false +toleration: + enabled: false +invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" ``` -You can use the command `helm status owdev -n openwhisk` to get a summary of the various -Kubernetes artifacts that make up your OpenWhisk deployment. Once the -`install-packages` Pod is in the Completed state, your OpenWhisk deployment -is ready to be used. +## Hints and Tips -Tip: If you notice errors or pods stuck in the pending state (`init-couchdb` +If you notice errors or pods stuck in the pending state (`init-couchdb` as an example), try running `kubectl get pvc --all-namespaces`. If you notice that claims are stuck in the Pending state, you may need to follow the workaround mentioned in this [Docker for Windows Github Issue](https://github.com/docker/for-win/issues/1758#issuecomment-376054370). -You are now ready to set up the wsk cli. Further instructions can be -[found here](https://github.com/apache/openwhisk-deploy-kube#https://github.com/apache/openwhisk-deploy-kube#configure-the-wsk-cli). -Follow the Docker for Windows instructions. - -## Hints and Tips - One nice feature of using Kubernetes in Docker, is that the containers being run in Kubernetes are also directly visible/accessible via the usual Docker commands. Furthermore, it is @@ -130,11 +110,3 @@ deployments of OpenWhisk. TLS termination will be handled by OpenWhisk's `nginx` service and will use self-signed certificates. You will need to invoke `wsk` with the `-i` command line argument to bypass certificate checking. - -The docker network is not exposed to the host on Windows. However, the -exposed ports for NodePort services are forwarded from localhost. -Therefore you must use different host names to connect to OpenWhisk -from outside the cluster (with the `wsk` cli) and from inside the -cluster (in `mycluster.yaml`). Continuing the example from above, -when setting the `--apihost` for the `wsk` cli, you would use -`localhost:31001`. diff --git a/docs/k8s-ibm-private.md b/docs/k8s-ibm-private.md deleted file mode 100644 index 21c12fc8..00000000 --- a/docs/k8s-ibm-private.md +++ /dev/null @@ -1,197 +0,0 @@ - - -# Deploying OpenWhisk on IBM Cloud Private (ICP) - -## Overview - -IBM Cloud Private (ICP) provides the core infrastructure needed to provision a -production-quality OpenWhisk installation. This document outlines -ICP-specific steps needed to provision that installation, and calls out -shortcuts that could be taken for development-grade installation. - -## Initial setup - -### Creating the Kubernetes Cluster - -Follow IBM Cloud Private instructions to provision your cluster. Include -GlusterFS provisioning, add -[dynamic NFS provisioning](./k8s-nfs-dynamic-storage.md), -or be prepared to provision volumes manually for OpenWhisk -(see [here](./configurationChoices.md#persistence)). - -### Configuring OpenWhisk - -#### Configuring Image Security - -IBM Cloud Private includes a provision for filtering the images that are -allowed to be deployed into a particular namespace. One _could_ disable this -capability for the OpenWhisk namespace, but initally it is best to define -a policy for the namespace: (In this case we assume the namespace is -`openwhisk`) - -```yaml -apiVersion: securityenforcement.admission.cloud.ibm.com/v1beta1 -kind: ImagePolicy -metadata: - name: openwhisk-image-policy - namespace: openwhisk -spec: - repositories: - - name: docker.io/openwhisk/* - policy: - va: - enabled: false - - name: docker.io/apache/couchdb:* - policy: - va: - enabled: false - - name: docker.io/nginx:* - policy: - va: - enabled: false - - name: docker.io/redis:* - policy: - va: - enabled: false - - name: docker.io/zookeeper:* - policy: - va: - enabled: false - - name: docker.io/wurstmeister/kafka:* - policy: - va: - enabled: false -``` - -#### Configuring Ingress - -An IBM Cloud Private cluster has full support for TLS -and can be configured with additional annotations to -fine tune ingress performance. - -A prerequisite for OpenWhisk TLS access via Ingress as currently configured -is a Fully Qualified Domain Name (FQDN) that can be resolved correctly from -within OpenWhisk and points to the SSL Ingress point, usually your load -balancer or proxy node. - -You will also need to create a TLS certificate to be used by the Ingress -controller for your domain. The YAML to create in Kubernetes is -(substituting the real values for ``): - -```yaml -apiVersion: certmanager.k8s.io/v1alpha1 -kind: Certificate -metadata: - name: openwhisk-tls-secret-1 - namespace: openwhisk -spec: - commonName: - dnsNames: - - - issuerRef: - kind: ClusterIssuer - name: icp-ca-issuer - secretName: openwhisk-tls-secret-1 -``` - -#### Putting it all together - -Now define `mycluster.yaml` as below (substituting the real values for -``). - -```yaml -whisk: - ingress: - apiHostName: - apiHostPort: 443 - apiHostProto: https - type: Standard - domain: - tls: - enabled: true - secretenabled: true - createsecret: false - secretname: openwhisk-tls-secret-1 - annotations: - # A blocking request is held open by the controller for slightly more than 60 seconds - # before it is responded to with HTTP status code 202 (accepted) and closed. - # Set to 75s to be on the safe side. - # See https://console.bluemix.net/docs/containers/cs_annotations.html#proxy-connect-timeout - # See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout - nginx.ingress.kubernetes.io/proxy-read-timeout: "75s" - - # Allow up to 50 MiB body size to support creation of large actions and large - # parameter sizes. - # See https://console.bluemix.net/docs/containers/cs_annotations.html#client-max-body-size - # See http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size - nginx.ingress.kubernetes.io/client-max-body-size: "size=50m" - - # Add the request_id, generated by nginx, to the request against the controllers. This id will be used as tid there. - # Note that the serviceName includes the argument to --name from the helm deploy command. (owdev in this example) - # https://console.bluemix.net/docs/containers/cs_annotations.html#proxy-add-headers - nginx.ingress.kubernetes.io/proxy-add-headers: | - serviceName=owdev-controller { - 'X-Request-ID' $request_id; - } - -k8s: - persistence: - hasDefaultStorageClass: false - explicitStorageClass: openwhisk -``` - -ICP does not (by default) provide a properly configured DefaultStorageClass, -instead you need to tell the Helm chart to use a storage class you've -defined (see Creating the Kubernetes Cluster -[above](#creating-the-kubernetes-cluster)). - -#### Don't want to deal with Ingress (or can't create an FQDN)? - -An alternative to the Ingress-based access model is to -use a NodePort. Use the IP address of any worker node in the cluster to -define `mycluster.yaml` as - -```yaml -whisk: - ingress: - type: NodePort - apiHostName: YOUR_WORKERS_PUBLIC_IP_ADDR - apiHostPort: 31001 - -nginx: - httpsNodePort: 31001 - -k8s: - persistence: - hasDefaultStorageClass: false - explicitStorageClass: openwhisk -``` - -ICP does not (by default) provide a properly configured DefaultStorageClass, -instead you need to tell the Helm chart to use a storage class you've -defined (see Creating the Kubernetes Cluster -[above](#creating-the-kubernetes-cluster)). - -## Hints and Tips - -On IBM Cloud Private clusters, you can configure OpenWhisk to integrate -with platform logging and monitoring services following the general -instructions for enabling these services for pods deployed on -Kubernetes. diff --git a/docs/k8s-ibm-public.md b/docs/k8s-ibm-public.md index 8dcd2265..0df98783 100644 --- a/docs/k8s-ibm-public.md +++ b/docs/k8s-ibm-public.md @@ -43,21 +43,18 @@ fine tune ingress performance. First, determine the values for and for your cluster by running the command: ``` -bx cs cluster-get +ibmcloud cs cluster get -c ``` The CLI output will look something like ``` -bx cs cluster-get +ibmcloud cs cluster get -c Retrieving cluster ... OK Name: -ID: b9c6b00dc0aa487f97123440b4895f2d -Created: 2017-04-26T19:47:08+0000 -State: normal -Master URL: https://169.57.40.165:1931 -Ingress subdomain: -Ingress secret: -Workers: 3 +... +Ingress Subdomain: +Ingress Secret: +... ``` As described in [IBM's ingress documentation](https://cloud.ibm.com/docs/containers/cs_ingress.html#ingress), @@ -67,65 +64,41 @@ a convention of using the namespace name as the subdomain name. So if you are deploying openwhisk into the `openwhisk` namespace, use `openwhisk` as your subdomain (as shown below in the example `mycluster.yaml`). -Now define `mycluster.yaml` as below (substituting the real values for -`` and ``). +A template [mycluster.yaml](../deploy/ibm-public/mycluster-iks.yaml] +for a standard deployment of OpenWhisk on IKS would be: ```yaml whisk: ingress: + # NOTE: Replace with your cluster's actual domain apiHostName: openwhisk. apiHostPort: 443 apiHostProto: https type: Standard + useInternally: true + # NOTE: Replace with your cluster's actual domain domain: openwhisk. tls: enabled: true secretenabled: true createsecret: false + # NOTE: Replace with your cluster's actual tlssecret secretname: annotations: - # A blocking request is held open by the controller for slightly more than 60 seconds - # before it is responded to with HTTP status code 202 (accepted) and closed. - # Set to 75s to be on the safe side. - # See https://console.bluemix.net/docs/containers/cs_annotations.html#proxy-connect-timeout - # See http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout - ingress.bluemix.net/proxy-read-timeout: "75s" - - # Allow up to 50 MiB body size to support creation of large actions and large - # parameter sizes. - # See https://console.bluemix.net/docs/containers/cs_annotations.html#client-max-body-size - # See http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size - ingress.bluemix.net/client-max-body-size: "size=50m" - - # Add the request_id, generated by nginx, to the request against the controllers. This id will be used as tid there. - # Note that the serviceName includes the argument to --name from the helm deploy command. (owdev in this example) - # https://console.bluemix.net/docs/containers/cs_annotations.html#proxy-add-headers - ingress.bluemix.net/proxy-add-headers: | - serviceName=owdev-controller { - 'X-Request-ID' $request_id; - } + kubernetes.io/ingress.class: public-iks-k8s-nginx + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_set_header X-Request-ID $request_id; + nginx.ingress.kubernetes.io/proxy-body-size: 50m + nginx.ingress.kubernetes.io/proxy-read-timeout: "75" invoker: containerFactory: impl: kubernetes - -k8s: - persistence: - hasDefaultStorageClass: false - explicitStorageClass: default ``` -Starting with IKS 1.11, the underlying container runtime is now -containerd instead of docker. As a result, you cannot use the -DockerContainerFactory on IKS and must use the -KubernetesContainerFactory. - -IKS does not provide a properly configured DefaultStorageClass, -instead you need to tell the Helm chart to use the `default` -StorageClassName as shown above. This StorageClass does have -a dynamic provisioner, so it is not necessary to manually create -the PersistentVolumes. Note that it is not unusual for it to take -several minutes for your PersistentVolumes to be created -(dependent resources will be in `Pending` state). +The underlying container runtime used by IKS is containerd. +Therefore, you cannot use the DockerContainerFactory on IKS and must +use the KubernetesContainerFactory. #### IBM Cloud Lite cluster @@ -133,7 +106,7 @@ The only available ingress method for an IBM Cloud Lite cluster is to use a NodePort. Obtain the Public IP address of the sole worker node by using the command ```shell -bx cs workers +ibmcloud cs workers ``` Then define `mycluster.yaml` as ```yaml @@ -142,24 +115,23 @@ whisk: type: NodePort apiHostName: YOUR_WORKERS_PUBLIC_IP_ADDR apiHostPort: 31001 + useInternally: true nginx: httpsNodePort: 31001 -k8s: - persistence: - hasDefaultStorageClass: false - explicitStorageClass: default +# disable affinity +affinity: + enabled: false +toleration: + enabled: false +invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" + # must use KCF as IKS uses containerd as its container runtime + containerFactory: + impl: "kubernetes" ``` -IKS does not provide a properly configured DefaultStorageClass, -instead you need to tell the Helm chart to use the `default` -StorageClassName as shown above. This StorageClass does have -a dynamic provisioner, so it is not necessary to manually create -the PersistentVolumes. Note that it is not unusual for it to take -several minutes for your PersistentVolumes to be created -(dependent resources will be in `Pending` state). - ## Hints and Tips On IBM Standard clusters, you can configure OpenWhisk to integrate diff --git a/docs/k8s-kind.md b/docs/k8s-kind.md index f220de9b..af25ec47 100644 --- a/docs/k8s-kind.md +++ b/docs/k8s-kind.md @@ -36,7 +36,7 @@ operate a default cluster with two virtual worker nodes. Download the latest stable release of `kind` for your platform from https://github.com/kubernetes-sigs/kind/releases. Our TravisCI testing -currently uses kind v0.10.0 on an ubuntu 18.04 host. +currently uses kind v0.11.1 on an ubuntu 18.04 host. ### Creating the Kubernetes Cluster @@ -44,59 +44,43 @@ On Linux, make sure your userid is in the `docker` group on the host machine. This will enable you to run `kind` without requiring `sudo` to gain `root` privileges. -Create a kind-cluster.yaml to configure your cluster. -```yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane -- role: worker - extraPortMappings: - - hostPort: 31001 - containerPort: 31001 -- role: worker -``` -The extraPortMappings stanza enables port forwarding -from the localhost to the in-cluster network. -This is required on MacOS, but to simplify the instructions -we use the same setup for all platforms. - -Now create your cluster with the command: -```shell -kind create cluster --config kind-cluster.yaml +We've provided a [script](./deploy/kind/start-kind.sh) +that you can use to bring up a kind cluster in a +reasonable configuration for OpenWhisk. The script +assumes that port 31001 is available on your machine +and can be used by openwhisk. To use a different port, +edit `deploy/kind/kind-cluster.yaml`. ``` - -Then label the two worker nodes so that one is reserved for the invoker -and the other will be used to run the rest of the OpenWhisk system. -```shell -kubectl label node kind-worker openwhisk-role=core -kubectl label node kind-worker2 openwhisk-role=invoker +./deploy/kind/start-kind.sh ``` ### Configuring OpenWhisk -To configure OpenWhisk, you first need to define a mycluster.yaml -that specifies the "inside the cluster" ingress information and -other system configuration. First, determine the internalIP of -a worker node with the command: -``` -kubectl describe node kind-worker | grep InternalIP: | awk '{print $2}' -``` -A mycluster.yaml for a standard deployment of OpenWhisk would look -like the below, replacing with its actual value: +Assuming you used the default port 31001 when starting kind, a +[mycluster.yaml](../deploy/kind/mycluster.yaml] +for a standard deployment of OpenWhisk would be: + ```yaml whisk: ingress: type: NodePort - apiHostName: + apiHostName: localhost apiHostPort: 31001 + useInternally: false + +nginx: + httpsNodePort: 31001 +# disable affinity +affinity: + enabled: false +toleration: + enabled: false invoker: + options: "-Dwhisk.kubernetes.user-pod-node-affinity.enabled=false" + # must use KCF as kind uses containerd as its container runtime containerFactory: impl: "kubernetes" - -nginx: - httpsNodePort: 31001 ``` Note that you must use the KubernetesContainerFactory when running OpenWhisk on `kind` because it is configured to use `containerd` @@ -104,8 +88,8 @@ as the underlying container engine. External to the Kubernetes cluster, for example when using the `wsk` cli, we will use the port forwarding configured by the `extraPortMappings` -in kind-cluster.yaml to allow the OpenWhisk apihost property -to be set to localhost:31001 +in [kind-cluster.yaml](../deploy/kind/kind-cluster.yaml) to allow the +OpenWhisk apihost property to be set to localhost:31001 ## Hints and Tips diff --git a/docs/k8s-technical-requirements.md b/docs/k8s-technical-requirements.md index d62831d6..d6e9200c 100644 --- a/docs/k8s-technical-requirements.md +++ b/docs/k8s-technical-requirements.md @@ -21,8 +21,8 @@ The Kubernetes cluster on which you are deploying OpenWhisk must meet the following requirements: -* [Kubernetes](https://github.com/kubernetes/kubernetes) version 1.16+. - Our automated testing currently covers Kubernetes versions 1.17, 1.18 and 1.19. +* [Kubernetes](https://github.com/kubernetes/kubernetes) version 1.19+. + Our automated testing currently covers Kubernetes versions 1.19, 1.20, and 1.21. * The ability to create Ingresses to make a Kubernetes service available outside of the cluster so you can actually use OpenWhisk. * Unless you disable persistence (see diff --git a/docs/okd-311.md b/docs/okd-311.md deleted file mode 100644 index a8ba7fe9..00000000 --- a/docs/okd-311.md +++ /dev/null @@ -1,79 +0,0 @@ - - -# Deploying OpenWhisk on OKD/OpenShift 3.11 - -## Overview - -The 3.11 version of OKD/OpenShift is based on Kubernetes 1.11. - -We assume you have an operational cluster that meets the -[technical requirements](okd-technical-requirements.md) and that you -have sufficient priveleges to perform the necessary `oc adm` -operations detailed below. - -## Initial Setup - -Create an openwhisk project (Kubernetes namespace) using the command -```shell -oc new-project openwhisk -``` - -Because OpenShift doesn’t allow arbitrary UIDs by default, execute the following commands: -```shell -oc adm policy add-scc-to-user anyuid -z default -oc adm policy add-scc-to-user privileged -z default -oc adm policy add-scc-to-user anyuid -z openwhisk-core -oc adm policy add-scc-to-user privileged -z openwhisk-core -oc adm policy add-scc-to-user anyuid -z owdev-init-sa -oc adm policy add-scc-to-user privileged -z owdev-init-sa -``` - -## Configuring OpenWhisk - -You must use the KubernetesContainerFactory on OKD/OpenShift. - -Here is a sample `mycluster.yaml`, where -should be replaced with the domain used for Routes in your cluster. -```yaml -whisk: - ingress: - type: OpenShift - apiHostName: openwhisk. - apiHostPort: 443 - apiHostProto: https - domain: openwhisk. - testing: - includeTests: false - -invoker: - containerFactory: - impl: kubernetes -`` - -## Limitations - -The nginx service is currently not deployed on OpenShift (problem -determining the appropriate value to use for `k8s.dns`, which is used to -set the resolver in `nginx-cm.yaml`). As a result, the namespace -prefixed 'vanity url' rewriting routes and the download of the cli/SDK -binaries is not currently supported when deploying on OpenShift. - -Smoketesting a deployment via `helm test` is not supported because -we did not use `helm install` to deploy the chart. diff --git a/docs/openshift-4.md b/docs/openshift-4.md new file mode 100644 index 00000000..08681c19 --- /dev/null +++ b/docs/openshift-4.md @@ -0,0 +1,126 @@ + + +# Deploying OpenWhisk on OpenShift 4.6 + +## Overview + +The 4.6 version of OpenShift is based on Kubernetes 1.19. + +We assume you have an operational cluster that meets the +[technical requirements](openshift-technical-requirements.md) and that you +have sufficient privileges to perform the necessary `oc adm` +operations detailed below. + +## Initial Setup + +Create an openwhisk project (Kubernetes namespace) using the command +```shell +oc new-project openwhisk +``` + +Because OpenShift doesn’t allow pods to run with arbitrary UIDs +by default, you will need to add adjust some policy options +before deploying OpenWhisk. Execute the following commands: +```shell +oc adm policy add-scc-to-user anyuid -z default +oc adm policy add-scc-to-user privileged -z default +oc adm policy add-scc-to-user anyuid -z openwhisk-core +oc adm policy add-scc-to-user privileged -z openwhisk-core +oc adm policy add-scc-to-user anyuid -z owdev-init-sa +oc adm policy add-scc-to-user privileged -z owdev-init-sa +``` + +## Configuring OpenWhisk + +You must use the KubernetesContainerFactory on OpenShift. + +### Red Hat OpenShift on IBM Cloud + +A Red Hat OpenShift on IBM Cloud cluster has full support for TLS +including a wild-card certificate for subdomains and can be configured +with additional annotations to fine tune ingress performance. + +First, determine the values for and for +your cluster by running the command: +``` +ibmcloud cs cluster get -c +``` +The CLI output will look something like +``` +ibmcloud cs cluster get -c +Retrieving cluster ... +OK +Name: +... +Ingress Subdomain: +Ingress Secret: +... +``` + +The ingress secret is not automatically copied to new OpenShift +projects. Before deploying OpenWhisk, you will need to copy the +ingress secret ( from the `openshift-ingress` namespace +to the `openwhisk` namespace. + +As described in [IBM's ingress documentation](https://cloud.ibm.com/docs/containers/cs_ingress.html#ingress), +to enable applications deployed in multiple namespaces to share the ingress resource, +you should use a unique subdomain name for each namespace. We suggest +a convention of using the namespace name as the subdomain name. So if you +are deploying openwhisk into the `openwhisk` namespace, use `openwhisk` +as your subdomain (as shown below in the example `mycluster.yaml`). + +A template [mycluster.yaml](../deploy/ibm-public/mycluster-roks.yaml] +for a standard deployment of OpenWhisk on ROKS would be: +```yaml +whisk: + ingress: + # NOTE: Replace with your cluster's actual domain + apiHostName: openwhisk. + apiHostPort: 443 + apiHostProto: https + type: Standard + useInternally: true + # NOTE: Replace with your cluster's actual domain + domain: openwhisk. + tls: + enabled: true + secretenabled: true + createsecret: false + # NOTE: Replace with your cluster's actual tlssecret + secretname: + annotations: + kubernetes.io/ingress.class: public-iks-k8s-nginx + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_set_header X-Request-ID $request_id; + nginx.ingress.kubernetes.io/proxy-body-size: 50m + nginx.ingress.kubernetes.io/proxy-read-timeout: "75" + +k8s: + dns: dns-default.openshift-dns + +invoker: + containerFactory: + impl: kubernetes +`` + +## Limitations + +No known limitations. diff --git a/docs/okd-technical-requirements.md b/docs/openshift-technical-requirements.md similarity index 84% rename from docs/okd-technical-requirements.md rename to docs/openshift-technical-requirements.md index c0a676fe..46c4b6d1 100644 --- a/docs/okd-technical-requirements.md +++ b/docs/openshift-technical-requirements.md @@ -17,12 +17,12 @@ # --> -# Technical Requirements for OKD/OpenShift +# Technical Requirements for OpenShift -The OKD/OpenShift cluster on which you are deploying OpenWhisk must meet +The OpenShift cluster on which you are deploying OpenWhisk must meet the following requirements: -* OKD/OpenShift version 3.11.x -* The ability to create routes to make a Kubernetes service +* OpenShift version 4.5 or newer (these instructions were tested on 4.5). +* The ability to create Ingresses to make a Kubernetes service available outside of the cluster so you can actually use OpenWhisk. * Unless you disable persistence (see [configurationChoices.md](configurationChoices.md)), diff --git a/docs/private-docker-registry.md b/docs/private-docker-registry.md index 69ffbc23..245b5dc0 100644 --- a/docs/private-docker-registry.md +++ b/docs/private-docker-registry.md @@ -42,7 +42,7 @@ By default, helm charts currently use docker hub to download images to deploy Op registry: name: "registry-name/" username: username - password: "Passowrd" + password: "Password" ``` > - enabling registry information will cause all your images to be pulled from private docker registry only. diff --git a/helm/openwhisk/Chart.yaml b/helm/openwhisk/Chart.yaml index 77f45c76..4bcff785 100644 --- a/helm/openwhisk/Chart.yaml +++ b/helm/openwhisk/Chart.yaml @@ -28,4 +28,4 @@ keywords: maintainers: - name: Apache OpenWhisk Community email: dev@openwhisk.apache.org -kubeVersion: ">=v1.16.0-r0" +kubeVersion: ">=v1.19.0-r0" diff --git a/helm/openwhisk/README.md b/helm/openwhisk/README.md index 1e6e8670..62b47a8a 100644 --- a/helm/openwhisk/README.md +++ b/helm/openwhisk/README.md @@ -38,7 +38,7 @@ In its default configuration, this chart will create the following Kubernetes re * apigateway, controller, couchdb, kafka, nginx, redis, zookeeper * OpenWhisk control plane Pods: * Deployments: apigateway, couchdb, nginx, redis - * SatefulSets: controller, invoker, kafka, zookeeper + * StatefulSets: controller, invoker, kafka, zookeeper * Persistent Volume Claims * couchdb-pvc * kafka-pvc @@ -56,7 +56,7 @@ The chart requires one or more Kubernetes worker nodes to be designated to be us ## Prerequisites -* Kubernetes 1.14 - 1.18.* +* Kubernetes 1.19+ ### Image Policy Requirements diff --git a/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh b/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh index 3993eedd..705fba86 100644 --- a/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh +++ b/helm/openwhisk/configMapFiles/tests/packageChecker/myTask.sh @@ -54,7 +54,6 @@ packageListingCheck "github" packageListingCheck "slack" packageListingCheck "utils" packageListingCheck "samples" -packageListingCheck "weather" packageListingCheck "websocket" # Check packages for installed event providers diff --git a/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh b/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh index d599f4a1..0ac526de 100644 --- a/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh +++ b/helm/openwhisk/configMapFiles/tests/smoketest/myTask.sh @@ -52,7 +52,7 @@ until $ACTION_LIST_PASSED; do done # next invoke the new hello world action via the CLI -echo "Inoking action via CLI" +echo "Invoking action via CLI" RESULT=$(wsk -i action invoke --blocking hello | grep "\"status\": \"success\"") if [ -z "$RESULT" ]; then echo "FAILED! Could not invoke hello action via CLI" @@ -62,6 +62,9 @@ fi # now run it as a web action echo "Invoking as web action" HELLO_URL=$(wsk -i action get hello --url | grep "https://") +if [ -z "$HELLO_URL" ]; then + HELLO_URL=$(wsk -i action get hello --url | grep "http://") +fi RESULT=$(wget --no-check-certificate -qO- $HELLO_URL | grep 'Hello world') if [ -z "$RESULT" ]; then echo "FAILED! Could not invoke hello as a web action" @@ -73,13 +76,16 @@ echo "Registering as an api" wsk -i api create /demo /hello get hello || (echo "FAILED: unable to create API"; exit 1) echo "Invoking action via the api" API_URL=$(wsk -i api list | grep hello | awk '{print $4}') -RESULT=$(wget --no-check-certificate -qO- "$API_URL" | grep 'Hello world') +echo "External api URL: $API_URL" +INTERNAL_URL=$(echo $API_URL | sed s#^http.*/api/#$WSK_API_HOST_URL/api/#) +echo "Internal api URL: $INTERNAL_URL" +RESULT=$(wget --no-check-certificate -qO- "$INTERNAL_URL" | grep 'Hello world') if [ -z "$RESULT" ]; then echo "FAILED! Could not invoke hello via apigateway" exit 1 fi -# now delete the resouces so the test could be run again +# now delete the resources so the test could be run again wsk -i api delete /demo || (echo "FAILED! failed to delete API"; exit 1) wsk -i action delete hello || (echo "FAILED! failed to delete action"; exit 1) diff --git a/helm/openwhisk/runtimes-minimal-travis.json b/helm/openwhisk/runtimes-minimal-travis.json index 716c427f..e30fe702 100644 --- a/helm/openwhisk/runtimes-minimal-travis.json +++ b/helm/openwhisk/runtimes-minimal-travis.json @@ -8,7 +8,7 @@ "image": { "prefix": "openwhisk", "name": "action-nodejs-v10", - "tag": "1.17.0" + "tag": "1.18.0" }, "deprecated": false, "attached": { @@ -22,8 +22,8 @@ "kind": "python:3", "image": { "prefix": "openwhisk", - "name": "python3action", - "tag": "1.15.0" + "name": "actionloop-python-v3.7", + "tag": "1.16.0" }, "deprecated": false } @@ -35,7 +35,7 @@ "image": { "prefix": "openwhisk", "name": "action-swift-v4.2", - "tag": "1.15.0" + "tag": "1.16.0" }, "deprecated": false, "attached": { diff --git a/helm/openwhisk/runtimes.json b/helm/openwhisk/runtimes.json index 167bb585..fd14f690 100644 --- a/helm/openwhisk/runtimes.json +++ b/helm/openwhisk/runtimes.json @@ -7,7 +7,7 @@ "image": { "prefix": "openwhisk", "name": "action-nodejs-v10", - "tag": "1.17.0" + "tag": "1.18.0" }, "deprecated": false, "attached": { @@ -27,7 +27,7 @@ "image": { "prefix": "openwhisk", "name": "action-nodejs-v12", - "tag": "1.17.0" + "tag": "1.18.0" }, "deprecated": false, "attached": { @@ -41,7 +41,7 @@ "image": { "prefix": "openwhisk", "name": "action-nodejs-v14", - "tag": "1.17.0" + "tag": "1.18.0" }, "deprecated": false, "attached": { @@ -70,8 +70,8 @@ "default": true, "image": { "prefix": "openwhisk", - "name": "python3action", - "tag": "1.15.0" + "name": "actionloop-python-v3.7", + "tag": "1.16.0" }, "deprecated": false, "attached": { @@ -87,7 +87,7 @@ "image": { "prefix": "openwhisk", "name": "action-swift-v4.2", - "tag": "1.15.0" + "tag": "1.16.0" }, "deprecated": false, "attached": { @@ -101,7 +101,21 @@ "image": { "prefix": "openwhisk", "name": "action-swift-v5.1", - "tag": "1.15.0" + "tag": "1.16.0" + }, + "deprecated": false, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + } + }, + { + "kind": "swift:5.3", + "default": false, + "image": { + "prefix": "openwhisk", + "name": "action-swift-v5.3", + "tag": "1.16.0" }, "deprecated": false, "attached": { @@ -117,7 +131,7 @@ "image": { "prefix": "openwhisk", "name": "java8action", - "tag": "1.15.0" + "tag": "1.16.0" }, "deprecated": false, "attached": { @@ -135,7 +149,7 @@ "image": { "prefix": "openwhisk", "name": "action-php-v7.3", - "tag": "1.15.0" + "tag": "1.16.0" }, "attached": { "attachmentName": "codefile", @@ -149,7 +163,21 @@ "image": { "prefix": "openwhisk", "name": "action-php-v7.4", - "tag": "1.15.0" + "tag": "1.16.0" + }, + "attached": { + "attachmentName": "codefile", + "attachmentType": "text/plain" + } + }, + { + "kind": "php:8.0", + "default": false, + "deprecated": false, + "image": { + "prefix": "openwhisk", + "name": "action-php-v7.4", + "tag": "1.16.0" }, "attached": { "attachmentName": "codefile", @@ -169,7 +197,7 @@ "image": { "prefix": "openwhisk", "name": "action-ruby-v2.5", - "tag": "1.15.0" + "tag": "1.16.0" } } ], @@ -185,7 +213,7 @@ "image": { "prefix": "openwhisk", "name": "action-golang-v1.15", - "tag": "1.17.0" + "tag": "1.18.0" } } ], @@ -201,7 +229,7 @@ "image": { "prefix": "openwhisk", "name": "action-rust-v1.34", - "tag": "1.1.0" + "tag": "1.2.0" } } ], diff --git a/helm/openwhisk/templates/NOTES.txt b/helm/openwhisk/templates/NOTES.txt index b8bb5609..fb190205 100644 --- a/helm/openwhisk/templates/NOTES.txt +++ b/helm/openwhisk/templates/NOTES.txt @@ -15,7 +15,7 @@ limitations under the License. */}} Apache OpenWhisk -Copyright 2016-2020 The Apache Software Foundation +Copyright 2016-2021 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/helm/openwhisk/templates/_affinity.tpl b/helm/openwhisk/templates/_affinity.tpl index ba0e4dec..61ebbcc3 100644 --- a/helm/openwhisk/templates/_affinity.tpl +++ b/helm/openwhisk/templates/_affinity.tpl @@ -61,7 +61,7 @@ nodeAffinity: {{- end -}} -{{/* Generic provder affinity */}} +{{/* Generic provider affinity */}} {{- define "openwhisk.affinity.provider" -}} # prefer to not run on an invoker node (only prefer because of single node clusters) nodeAffinity: diff --git a/helm/openwhisk/templates/_helpers.tpl b/helm/openwhisk/templates/_helpers.tpl index 108f87b1..6f441865 100644 --- a/helm/openwhisk/templates/_helpers.tpl +++ b/helm/openwhisk/templates/_helpers.tpl @@ -268,17 +268,17 @@ imagePullSecrets: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_proto + key: whisk_internal_api_host_proto - name: "WHISK_API_HOST_PORT" valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_port + key: whisk_internal_api_host_port - name: "WHISK_API_HOST_NAME" valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_name + key: whisk_internal_api_host_name {{- end -}} {{/* Environment variables required for invoker containerpool/containerfactory configuration */}} diff --git a/helm/openwhisk/templates/_readiness.tpl b/helm/openwhisk/templates/_readiness.tpl index e5bc7d50..da6a0fec 100644 --- a/helm/openwhisk/templates/_readiness.tpl +++ b/helm/openwhisk/templates/_readiness.tpl @@ -33,10 +33,9 @@ {{/* Init container that waits for kafka to be ready */}} {{- define "openwhisk.readiness.waitForKafka" -}} - name: "wait-for-kafka" - image: "{{- .Values.docker.registry.name -}}{{- .Values.busybox.imageName -}}:{{- .Values.busybox.imageTag -}}" + image: "{{- .Values.docker.registry.name -}}{{- .Values.utility.imageName -}}:{{- .Values.utility.imageTag -}}" imagePullPolicy: "IfNotPresent" - # TODO: I haven't found an easy external test to determine that kafka is up, so as a hack we wait for zookeeper and then sleep for 10 seconds and cross our fingers! - command: ["sh", "-c", 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 {{ include "openwhisk.zookeeper_zero_host" . }} {{ .Values.zookeeper.port }}); if [ "$OK" == "imok" ]; then result=0; echo "zookeeper returned imok!"; else echo waiting for zookeeper to be ready; sleep 1; fi done; echo "Zookeeper is up; will wait for 10 seconds to give kafka time to initialize"; sleep 10;'] + command: ["sh", "-c", 'cacert="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"; while true; do rc=$(curl -sS --cacert $cacert --header "Authorization: Bearer $token" https://kubernetes.default.svc/api/v1/namespaces/{{ .Release.Namespace }}/endpoints/{{ .Release.Name }}-kafka | jq -r ".subsets[].addresses | length"); echo "num ready kafka endpoints is $rc"; if [ $rc -gt 0 ]; then echo "Success: ready kafka endpoint!"; break; fi; echo "kafka not ready yet; sleeping for 3 seconds"; sleep 3; done;'] {{- end -}} {{/* Init container that waits for zookeeper to be ready */}} diff --git a/helm/openwhisk/templates/apigateway-pod.yaml b/helm/openwhisk/templates/apigateway-pod.yaml index 32229e18..aafe4b4a 100644 --- a/helm/openwhisk/templates/apigateway-pod.yaml +++ b/helm/openwhisk/templates/apigateway-pod.yaml @@ -69,12 +69,12 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_url + key: whisk_external_api_host_url - name: "BACKEND_HOST" valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_url + key: whisk_internal_api_host_url {{- if .Values.apigw.secure }} securityContext: capabilities: diff --git a/helm/openwhisk/templates/controller-pod.yaml b/helm/openwhisk/templates/controller-pod.yaml index 27d56e24..04e2ca3c 100644 --- a/helm/openwhisk/templates/controller-pod.yaml +++ b/helm/openwhisk/templates/controller-pod.yaml @@ -143,7 +143,7 @@ spec: # specific controller arguments - name: "CONTROLLER_OPTS" - value: "{{ .Values.controller.options }} {{ if .Values.controller.lean }} {{ include "openwhisk.invoker.add_opts" . }} {{ end }}" + value: "{{ .Values.controller.options }} {{ if .Values.controller.lean }} {{ .Values.invoker.options }} {{ include "openwhisk.invoker.add_opts" . }} {{ end }}" # action runtimes - name: "RUNTIMES_MANIFEST" diff --git a/helm/openwhisk/templates/couchdb-pod.yaml b/helm/openwhisk/templates/couchdb-pod.yaml index 01f7af75..68917196 100644 --- a/helm/openwhisk/templates/couchdb-pod.yaml +++ b/helm/openwhisk/templates/couchdb-pod.yaml @@ -28,6 +28,10 @@ spec: selector: matchLabels: name: {{ .Release.Name }}-couchdb + {{- if .Values.k8s.persistence.enabled }} + strategy: + type: "Recreate" + {{- end }} template: metadata: labels: diff --git a/helm/openwhisk/templates/frontdoor-ingress.yaml b/helm/openwhisk/templates/frontdoor-ingress.yaml index e00a0231..bbdf68c8 100644 --- a/helm/openwhisk/templates/frontdoor-ingress.yaml +++ b/helm/openwhisk/templates/frontdoor-ingress.yaml @@ -16,7 +16,7 @@ # {{- if eq .Values.whisk.ingress.type "Standard" }} -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ .Release.Name }}-ingress @@ -31,7 +31,7 @@ spec: tls: - hosts: - {{ .Values.whisk.ingress.domain }} - {{- if .Values.whisk.ingress.tls.secretenabled }} + {{- if ne .Values.whisk.ingress.tls.secretname "" }} secretName: {{ .Values.whisk.ingress.tls.secretname | quote }} {{- end }} {{- end }} @@ -40,30 +40,48 @@ spec: http: paths: - path: /api/v1/web + pathType: Prefix backend: - serviceName: {{ .Release.Name }}-controller - servicePort: http + service: + name: {{ .Release.Name }}-controller + port: + name: http - path: /api/v1 + pathType: Prefix backend: - serviceName: {{ .Release.Name }}-controller - servicePort: http + service: + name: {{ .Release.Name }}-controller + port: + name: http # API GW generated API invocation - path: /api + pathType: Prefix backend: - serviceName: {{ .Release.Name }}-apigateway - servicePort: mgmt + service: + name: {{ .Release.Name }}-apigateway + port: + name: mgmt # API GW health status - path: /v1/health-check + pathType: Prefix backend: - serviceName: {{ .Release.Name }}-apigateway - servicePort: api + service: + name: {{ .Release.Name }}-apigateway + port: + name: api # API management REST APIs - path: /v2 + pathType: Prefix backend: - serviceName: {{ .Release.Name }}-apigateway - servicePort: api + service: + name: {{ .Release.Name }}-apigateway + port: + name: api - path: / + pathType: Prefix backend: - serviceName: {{ .Release.Name }}-nginx - servicePort: http + service: + name: {{ .Release.Name }}-nginx + port: + name: http {{- end }} diff --git a/helm/openwhisk/templates/frontdoor-routes.yaml b/helm/openwhisk/templates/frontdoor-routes.yaml deleted file mode 100644 index 305e23b6..00000000 --- a/helm/openwhisk/templates/frontdoor-routes.yaml +++ /dev/null @@ -1,114 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{{- if eq .Values.whisk.ingress.type "OpenShift" }} -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: {{ .Release.Name }}-route-c1 -spec: - host: {{ .Values.whisk.ingress.domain }} - path: /api/v1/web - port: - targetPort: http - tls: - termination: edge - to: - kind: Service - name: {{ .Release.Name }}-controller - weight: 100 ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: {{ .Release.Name }}-route-c2 -spec: - host: {{ .Values.whisk.ingress.domain }} - path: /api/v1 - port: - targetPort: http - tls: - termination: edge - to: - kind: Service - name: {{ .Release.Name }}-controller - weight: 100 ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: {{ .Release.Name }}-route-a1 -spec: - host: {{ .Values.whisk.ingress.domain }} - path: /api - port: - targetPort: http - tls: - termination: edge - to: - kind: Service - name: {{ .Release.Name }}-apigateway - weight: 100 ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: {{ .Release.Name }}-route-a2 -spec: - host: {{ .Values.whisk.ingress.domain }} - path: /v1/health-check - port: - targetPort: http - tls: - termination: edge - to: - kind: Service - name: {{ .Release.Name }}-apigateway - weight: 100 ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: {{ .Release.Name }}-route-a3 -spec: - host: {{ .Values.whisk.ingress.domain }} - path: /v2 - port: - targetPort: http - tls: - termination: edge - to: - kind: Service - name: {{ .Release.Name }}-apigateway - weight: 100 ---- -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: {{ .Release.Name }}-route-r -spec: - host: {{ .Values.whisk.ingress.domain }} - path: / - port: - targetPort: http - tls: - termination: edge - to: - kind: Service - name: {{ .Release.Name }}-controller - weight: 100 -{{- end }} diff --git a/helm/openwhisk/templates/frontdoor-secrets.yaml b/helm/openwhisk/templates/frontdoor-secrets.yaml index 583ee814..4043a261 100644 --- a/helm/openwhisk/templates/frontdoor-secrets.yaml +++ b/helm/openwhisk/templates/frontdoor-secrets.yaml @@ -25,7 +25,7 @@ metadata: {{ include "openwhisk.label_boilerplate" . | indent 4 }} type: {{ .Values.whisk.ingress.tls.secrettype | quote }} data: - tls.crt: {{ .Values.whisk.ingress.tls.crt }} - tls.key: {{ .Values.whisk.ingress.tls.key }} + tls.crt: {{ .Values.whisk.ingress.tls.crt | b64enc }} + tls.key: {{ .Values.whisk.ingress.tls.key | b64enc }} {{- end}} {{- end }} diff --git a/helm/openwhisk/templates/gen-certs-cm.yaml b/helm/openwhisk/templates/gen-certs-cm.yaml index b74b5689..65f90385 100644 --- a/helm/openwhisk/templates/gen-certs-cm.yaml +++ b/helm/openwhisk/templates/gen-certs-cm.yaml @@ -15,6 +15,7 @@ # limitations under the License. # +{{- if or (eq .Values.whisk.ingress.type "NodePort") (eq .Values.whisk.ingress.type "LoadBalancer") }} apiVersion: v1 kind: ConfigMap metadata: @@ -32,3 +33,4 @@ data: {{- else }} {{ (.Files.Glob "configMapFiles/genCerts/gencerts.sh").AsConfig | indent 2 }} {{- end }} +{{- end }} diff --git a/helm/openwhisk/templates/gen-certs-job.yaml b/helm/openwhisk/templates/gen-certs-job.yaml index c72ecd51..e5d14831 100644 --- a/helm/openwhisk/templates/gen-certs-job.yaml +++ b/helm/openwhisk/templates/gen-certs-job.yaml @@ -15,6 +15,7 @@ # limitations under the License. # +{{- if or (eq .Values.whisk.ingress.type "NodePort") (eq .Values.whisk.ingress.type "LoadBalancer") }} {{- if not .Values.nginx.certificate.external }} apiVersion: batch/v1 kind: Job @@ -55,5 +56,6 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_name + key: whisk_external_api_host_name +{{- end }} {{- end }} diff --git a/helm/openwhisk/templates/install-packages-job.yaml b/helm/openwhisk/templates/install-packages-job.yaml index 71b2d37a..fc7cb304 100644 --- a/helm/openwhisk/templates/install-packages-job.yaml +++ b/helm/openwhisk/templates/install-packages-job.yaml @@ -37,7 +37,9 @@ spec: configMap: name: {{ .Release.Name }}-install-packages-cm initContainers: -{{- if not .Values.controller.lean }} +{{- if .Values.controller.lean }} +{{ include "openwhisk.readiness.waitForController" . | indent 6 }} +{{- else }} {{ include "openwhisk.readiness.waitForHealthyInvoker" . | indent 6 }} {{ end }} {{ include "openwhisk.docker.imagePullSecrets" . | indent 6 }} @@ -60,12 +62,12 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_nameAndPort + key: whisk_internal_api_host_nameAndPort - name: "WHISK_API_HOST_URL" valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_url + key: whisk_internal_api_host_url # apigateway configuration (for installing routemgmt actions) - name: "WHISK_SYSTEM_NAMESPACE" valueFrom: diff --git a/helm/openwhisk/templates/kafka-pod.yaml b/helm/openwhisk/templates/kafka-pod.yaml index a4741f4a..283e53e1 100644 --- a/helm/openwhisk/templates/kafka-pod.yaml +++ b/helm/openwhisk/templates/kafka-pod.yaml @@ -77,6 +77,21 @@ spec: resources: {{ toYaml .Values.kafka.resources | indent 10 }} {{- end }} + livenessProbe: + tcpSocket: + port: {{ .Values.kafka.port }} + initialDelaySeconds: {{ .Values.probes.kafka.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.kafka.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.probes.kafka.livenessProbe.timeoutSeconds }} + readinessProbe: + initialDelaySeconds: {{ .Values.probes.kafka.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.kafka.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.probes.kafka.readinessProbe.timeoutSeconds }} + exec: + command: + - /opt/kafka/bin/kafka-topics.sh + - localhost:{{ .Values.kafka.port }} + - --version env: - name: "HOSTNAME_COMMAND" value: "hostname -f" diff --git a/helm/openwhisk/templates/nginx-cm.yaml b/helm/openwhisk/templates/nginx-cm.yaml index 77240d1f..e25ca369 100644 --- a/helm/openwhisk/templates/nginx-cm.yaml +++ b/helm/openwhisk/templates/nginx-cm.yaml @@ -15,7 +15,6 @@ # limitations under the License. # -{{- if ne .Values.whisk.ingress.type "OpenShift" }} apiVersion: v1 kind: ConfigMap metadata: @@ -59,13 +58,16 @@ data: server { listen 80; +{{- if or (eq .Values.whisk.ingress.type "NodePort") (eq .Values.whisk.ingress.type "LoadBalancer") }} listen 443 default ssl; +{{- end }} # match namespace, note while OpenWhisk allows a richer character set for a # namespace, not all those characters are permitted in the (sub)domain name; # if namespace does not match, no vanity URL rewriting takes place. server_name ~^(?[0-9a-zA-Z-]+)\.{{ .Values.whisk.ingress.apiHostName }}$; +{{- if or (eq .Values.whisk.ingress.type "NodePort") (eq .Values.whisk.ingress.type "LoadBalancer") }} ssl_session_cache shared:SSL:1m; ssl_session_timeout 10m; ssl_certificate /etc/nginx/certs/tls.crt; @@ -81,6 +83,7 @@ data: ssl_prefer_server_ciphers on; proxy_ssl_session_reuse on; proxy_ssl_verify off; +{{- end }} # Hack to convince nginx to dynamically resolve the dns entries. resolver {{ .Values.k8s.dns }}; @@ -169,4 +172,3 @@ data: } } } -{{- end }} diff --git a/helm/openwhisk/templates/nginx-pod.yaml b/helm/openwhisk/templates/nginx-pod.yaml index 8e7bb08d..a6ad9538 100644 --- a/helm/openwhisk/templates/nginx-pod.yaml +++ b/helm/openwhisk/templates/nginx-pod.yaml @@ -15,7 +15,6 @@ # limitations under the License. # -{{- if ne .Values.whisk.ingress.type "OpenShift" }} apiVersion: apps/v1 kind: Deployment metadata: @@ -48,6 +47,7 @@ spec: {{- end }} volumes: + {{- if or (eq .Values.whisk.ingress.type "NodePort") (eq .Values.whisk.ingress.type "LoadBalancer") }} {{- if .Values.nginx.certificate.external }} - name: nginx-certs configMap: @@ -57,6 +57,7 @@ spec: secret: secretName: {{ .Release.Name }}-nginx {{- end }} + {{- end }} - name: nginx-conf configMap: name: {{ .Release.Name }}-nginx @@ -73,7 +74,7 @@ spec: ports: - name: http containerPort: {{ .Values.nginx.httpPort }} - - name: http-api + - name: https containerPort: {{ .Values.nginx.httpsPort }} {{- if .Values.nginx.resources }} resources: @@ -83,8 +84,9 @@ spec: - name: nginx-conf mountPath: "/etc/nginx/nginx.conf" subPath: "nginx.conf" + {{- if or (eq .Values.whisk.ingress.type "NodePort") (eq .Values.whisk.ingress.type "LoadBalancer") }} - name: nginx-certs mountPath: "/etc/nginx/certs" + {{- end }} - name: logs mountPath: "/logs" -{{- end }} diff --git a/helm/openwhisk/templates/nginx-svc.yaml b/helm/openwhisk/templates/nginx-svc.yaml index e67bc616..b82b2748 100644 --- a/helm/openwhisk/templates/nginx-svc.yaml +++ b/helm/openwhisk/templates/nginx-svc.yaml @@ -15,7 +15,6 @@ # limitations under the License. # -{{- if ne .Values.whisk.ingress.type "OpenShift" }} apiVersion: v1 kind: Service metadata: @@ -40,12 +39,14 @@ spec: ports: - port: {{ .Values.nginx.httpPort }} name: http + targetPort: http - port: {{ .Values.nginx.httpsPort }} {{- if eq .Values.whisk.ingress.type "NodePort" }} nodePort: {{ .Values.nginx.httpsNodePort }} {{- end }} - name: https-api + name: https {{- if eq .Values.whisk.ingress.awsSSL "true"}} targetPort: http + {{- else }} + targetPort: https {{- end}} -{{- end }} diff --git a/helm/openwhisk/templates/ow-whisk-cm.yaml b/helm/openwhisk/templates/ow-whisk-cm.yaml index 2e20623f..27b614d8 100644 --- a/helm/openwhisk/templates/ow-whisk-cm.yaml +++ b/helm/openwhisk/templates/ow-whisk-cm.yaml @@ -26,24 +26,21 @@ data: whisk_info_buildNo: {{ .Values.whisk.versions.openwhisk.buildNo | quote }} whisk_cli_version_tag: {{ .Values.whisk.versions.openwhiskCli.tag | quote }} whisk_system_namespace: {{ .Values.whisk.systemNameSpace | quote }} -{{- if eq .Values.whisk.ingress.type "LoadBalancer" }} -{{- if eq .Values.whisk.ingress.awsSSL "true" }} - whisk_api_host_proto: "http" - whisk_api_host_port: {{ .Values.nginx.httpPort | quote }} - whisk_api_host_name: "{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}" - whisk_api_host_nameAndPort: "{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}:{{ .Values.nginx.httpPort }}" - whisk_api_host_url: "http://{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}:{{ .Values.nginx.httpPort }}" + whisk_external_api_host_proto: {{ .Values.whisk.ingress.apiHostProto | quote }} + whisk_external_api_host_port: {{ .Values.whisk.ingress.apiHostPort | quote }} + whisk_external_api_host_name: {{ .Values.whisk.ingress.apiHostName | quote }} + whisk_external_api_host_nameAndPort: "{{ .Values.whisk.ingress.apiHostName }}:{{ .Values.whisk.ingress.apiHostPort }}" + whisk_external_api_host_url: "{{ .Values.whisk.ingress.apiHostProto }}://{{ .Values.whisk.ingress.apiHostName }}:{{ .Values.whisk.ingress.apiHostPort }}" +{{- if .Values.whisk.ingress.useInternally }} + whisk_internal_api_host_proto: {{ .Values.whisk.ingress.apiHostProto | quote }} + whisk_internal_api_host_port: {{ .Values.whisk.ingress.apiHostPort | quote }} + whisk_internal_api_host_name: {{ .Values.whisk.ingress.apiHostName | quote }} + whisk_internal_api_host_nameAndPort: "{{ .Values.whisk.ingress.apiHostName }}:{{ .Values.whisk.ingress.apiHostPort }}" + whisk_internal_api_host_url: "{{ .Values.whisk.ingress.apiHostProto }}://{{ .Values.whisk.ingress.apiHostName }}:{{ .Values.whisk.ingress.apiHostPort }}" {{- else }} - whisk_api_host_proto: "https" - whisk_api_host_port: {{ .Values.nginx.httpsPort | quote }} - whisk_api_host_name: "{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}" - whisk_api_host_nameAndPort: "{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}:{{ .Values.nginx.httpsPort }}" - whisk_api_host_url: "https://{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}:{{ .Values.nginx.httpsPort }}" -{{- end }} -{{- else }} - whisk_api_host_proto: {{ .Values.whisk.ingress.apiHostProto | quote }} - whisk_api_host_port: {{ .Values.whisk.ingress.apiHostPort | quote }} - whisk_api_host_name: {{ .Values.whisk.ingress.apiHostName | quote }} - whisk_api_host_nameAndPort: "{{ .Values.whisk.ingress.apiHostName }}:{{ .Values.whisk.ingress.apiHostPort }}" - whisk_api_host_url: "{{ .Values.whisk.ingress.apiHostProto }}://{{ .Values.whisk.ingress.apiHostName }}:{{ .Values.whisk.ingress.apiHostPort }}" + whisk_internal_api_host_proto: "http" + whisk_internal_api_host_port: {{ .Values.nginx.httpPort | quote }} + whisk_internal_api_host_name: "{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}" + whisk_internal_api_host_nameAndPort: "{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}:{{ .Values.nginx.httpPort }}" + whisk_internal_api_host_url: "http://{{ .Release.Name }}-nginx.{{ .Release.Namespace }}.svc.{{ .Values.k8s.domain }}:{{ .Values.nginx.httpPort }}" {{- end }} diff --git a/helm/openwhisk/templates/provider-alarm-pod.yaml b/helm/openwhisk/templates/provider-alarm-pod.yaml index 738170ae..aebe0f64 100644 --- a/helm/openwhisk/templates/provider-alarm-pod.yaml +++ b/helm/openwhisk/templates/provider-alarm-pod.yaml @@ -28,6 +28,10 @@ spec: selector: matchLabels: name: {{ .Release.Name }}-alarmprovider + {{- if .Values.k8s.persistence.enabled }} + strategy: + type: "Recreate" + {{- end }} template: metadata: labels: @@ -93,12 +97,12 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_nameAndPort + key: whisk_internal_api_host_nameAndPort - name: "ENDPOINT_AUTH" valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_nameAndPort + key: whisk_internal_api_host_nameAndPort volumeMounts: - name: alarm-logs mountPath: /logs diff --git a/helm/openwhisk/templates/provider-kafka-pod.yaml b/helm/openwhisk/templates/provider-kafka-pod.yaml index 2688f42f..cb8a82ba 100644 --- a/helm/openwhisk/templates/provider-kafka-pod.yaml +++ b/helm/openwhisk/templates/provider-kafka-pod.yaml @@ -81,10 +81,10 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_nameAndPort + key: whisk_internal_api_host_nameAndPort - name: "ENDPOINT_AUTH" valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_nameAndPort + key: whisk_internal_api_host_nameAndPort {{- end }} diff --git a/helm/openwhisk/templates/redis-pod.yaml b/helm/openwhisk/templates/redis-pod.yaml index d205f2ec..781ef5c1 100644 --- a/helm/openwhisk/templates/redis-pod.yaml +++ b/helm/openwhisk/templates/redis-pod.yaml @@ -28,6 +28,10 @@ spec: selector: matchLabels: name: {{ .Release.Name }}-redis + {{- if .Values.k8s.persistence.enabled }} + strategy: + type: "Recreate" + {{- end }} template: metadata: labels: diff --git a/helm/openwhisk/templates/tests/package-checker-pod.yaml b/helm/openwhisk/templates/tests/package-checker-pod.yaml index 72432a1b..f3f12ab1 100644 --- a/helm/openwhisk/templates/tests/package-checker-pod.yaml +++ b/helm/openwhisk/templates/tests/package-checker-pod.yaml @@ -52,7 +52,7 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_url + key: whisk_internal_api_host_url # Which of the providers was installed (and thus should be checked)? - name: "OW_INSTALL_ALARM_PROVIDER" diff --git a/helm/openwhisk/templates/tests/smoketest-pod.yaml b/helm/openwhisk/templates/tests/smoketest-pod.yaml index 30f64332..c1a93640 100644 --- a/helm/openwhisk/templates/tests/smoketest-pod.yaml +++ b/helm/openwhisk/templates/tests/smoketest-pod.yaml @@ -52,5 +52,5 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_url + key: whisk_internal_api_host_url {{- end }} diff --git a/helm/openwhisk/templates/tests/systemtest-pod.yaml b/helm/openwhisk/templates/tests/systemtest-pod.yaml index 032b2990..f8c8ca5c 100644 --- a/helm/openwhisk/templates/tests/systemtest-pod.yaml +++ b/helm/openwhisk/templates/tests/systemtest-pod.yaml @@ -52,7 +52,7 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-whisk.config - key: whisk_api_host_url + key: whisk_internal_api_host_url - name: "OW_GIT_TAG_OPENWHISK" value: {{ .Values.whisk.versions.openwhisk.gitTag | quote }} {{- end }} diff --git a/helm/openwhisk/values.schema.json b/helm/openwhisk/values.schema.json index 288e0ed4..4f1d2ff9 100644 --- a/helm/openwhisk/values.schema.json +++ b/helm/openwhisk/values.schema.json @@ -111,15 +111,16 @@ "apiHostName": { "type": "string", "description": "The external hostname or IP address used to access the Ingress of your Kubernetes cluster" }, "apiHostPort": { "type": "integer", "minimum": 0, "description": "The external port used to access the Ingress of your Kubernetes cluster" }, "apiHostProto": { "type": "string", "enum": ["http", "https"], "description": "The protocol to be used to connect to the Ingress of your Kubernetes cluster" }, - "type": { "type": "string", "enum": ["NodePort", "Standard", "LoadBalancer", "OpenShift"], "description": "The type of Ingress being deployed" }, + "type": { "type": "string", "enum": ["NodePort", "Standard", "LoadBalancer"], "description": "The type of Ingress being deployed" }, "annotations": { "type": "object", "description": "Annotations to add to Ingress resource. Specify as a list of key: value pairs" }, "domain": { "type": "string", "description": "The Fully Qualified Host Name for the Ingress domain" }, + "useInternally": { "type": "boolean", "description": "Should the external ingress be used for operations from inside the cluster" }, "tls": { "type": "object", "properties": { "enabled": { "type": "boolean" }, - "secretenabled": { "type": "boolean" }, "createsecret": { "type": "boolean" }, + "secretname": { "type": "string" }, "secrettype": { "type": "string" }, "crt": { "type": "string" }, "key": { "type": "string" } @@ -139,8 +140,8 @@ "limits": { "type": "object", "properties": { - "actionsInvokesPerminute": { "type": "number", "minimum": 0, "descrption": "The maximum number of action invocations per minute by a single namespace" }, - "actionsInvokesConcurrent": { "type": "number", "minimum": 0, "descrption": "The maximum number of concurrent action invocations by a single namespace" }, + "actionsInvokesPerminute": { "type": "number", "minimum": 0, "description": "The maximum number of action invocations per minute by a single namespace" }, + "actionsInvokesConcurrent": { "type": "number", "minimum": 0, "description": "The maximum number of concurrent action invocations by a single namespace" }, "triggersFiresPerminute": { "type": "number", "minimum": 0, "description": "The maximum triggers fired per minute for a single namespace" }, "actionsSequenceMaxLength": { "type": "number", "minimum": 0, "description": "The maximum length of an action sequence" }, "actions": { diff --git a/helm/openwhisk/values.yaml b/helm/openwhisk/values.yaml index 76fdf80a..ba024762 100644 --- a/helm/openwhisk/values.yaml +++ b/helm/openwhisk/values.yaml @@ -26,7 +26,6 @@ # docs/k8s-docker-for-mac.md # docs/k8s-aws.md # docs/k8s-ibm-public.md -# docs/k8s-ibm-private.md # docs/k8s-google.md # docs/k8s-diy.md (for do-it-yourself clusters). # @@ -48,7 +47,6 @@ whisk: # See the "Configuring OpenWhisk section" of the docs/k8s-*.md that matches # your cluster type for details on what values to provide and how to get them. ingress: - awsSSL: "false" apiHostName: "" apiHostPort: 31001 apiHostProto: "https" @@ -56,9 +54,10 @@ whisk: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "50m" domain: "domain" + awsSSL: "false" + useInternally: false tls: enabled: false - secretenabled: false createsecret: false secretname: "ow-ingress-tls-secret" secrettype: "type" @@ -135,7 +134,7 @@ whisk: openwhiskCli: tag: "1.1.0" openwhiskCatalog: - gitTag: "0.11.0" + gitTag: "1.0.0" openwhiskPackageAlarms: gitTag: "2.2.0" openwhiskPackageKafka: @@ -457,6 +456,15 @@ probes: initialDelaySeconds: 5 periodSeconds: 10 timeoutSeconds: 1 + kafka: + livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 1 + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 controller: livenessProbe: initialDelaySeconds: 10 diff --git a/tools/travis/deploy-chart.sh b/tools/travis/deploy-chart.sh index 9778e518..e4bcc583 100755 --- a/tools/travis/deploy-chart.sh +++ b/tools/travis/deploy-chart.sh @@ -171,35 +171,18 @@ OW_INCLUDE_SYSTEM_TESTS=${OW_INCLUDE_SYSTEM_TESTS:="false"} # Default timeout limit to 60 steps TIMEOUT_STEP_LIMIT=${TIMEOUT_STEP_LIMIT:=60} -# Label nodes for affinity. -# For DockerContainerFactory, at least one must be labeled as an invoker. -echo "Labeling nodes with openwhisk-role assignments" -kubectl label nodes kind-worker openwhisk-role=core -kubectl label nodes kind-worker2 openwhisk-role=invoker - # Create namespace kubectl create namespace openwhisk -# Configure a NodePort Ingress assuming kind conventions. -# Use kind-worker as the ingress, since we labeled it as our core node above. -# (But using kind-worker2 would also work because Kubernetes -# exposes the same NodePort service on all worker nodes.) -WSK_PORT=31001 -WSK_HOST=$(kubectl describe node kind-worker | grep InternalIP: | awk '{print $2}') -if [ -z "$WSK_HOST" ]; then - echo "FAILED! Could not determine value for WSK_HOST" - exit 1 -fi +# Default to kind conventions of using localhost:31001 +WSK_PORT=${WSK_PORT:=31001} +WSK_HOST=${WSK_HOST:=localhost} # Deploy OpenWhisk using Helm cd $ROOTDIR -cat > mycluster.yaml < ow-config.yaml < Date: Wed, 18 Aug 2021 16:45:22 -0400 Subject: [PATCH 17/18] only run seccomp pods on invoker nodes --- helm/openwhisk/templates/seccomp-pod.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helm/openwhisk/templates/seccomp-pod.yaml b/helm/openwhisk/templates/seccomp-pod.yaml index 8a4b6626..6c897e1f 100644 --- a/helm/openwhisk/templates/seccomp-pod.yaml +++ b/helm/openwhisk/templates/seccomp-pod.yaml @@ -35,3 +35,10 @@ spec: - name: profiles configMap: name: {{ .Release.Name }}-seccomp-profile +{{- if .Values.affinity.enabled }} + affinity: +{{ include "openwhisk.affinity.invoker" . | indent 8 }} +{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }} +{{ include "openwhisk.affinity.selfAntiAffinity" ( printf "%s-invoker" .Release.Name ) | indent 8 }} +{{- end }} +{{- end }} From fde6deb18fac3ce508b27838c2be69a52709b1d1 Mon Sep 17 00:00:00 2001 From: Mike Ludwig Date: Thu, 19 Aug 2021 16:25:59 -0400 Subject: [PATCH 18/18] remove dockerlogin --- helm/openwhisk/templates/_helpers.tpl | 5 ----- helm/openwhisk/templates/invoker-cm.yaml | 1 - helm/openwhisk/templates/invoker-pod.yaml | 3 --- helm/openwhisk/templates/seccomp-pod.yaml | 7 ------- 4 files changed, 16 deletions(-) diff --git a/helm/openwhisk/templates/_helpers.tpl b/helm/openwhisk/templates/_helpers.tpl index 6f441865..847ef8e2 100644 --- a/helm/openwhisk/templates/_helpers.tpl +++ b/helm/openwhisk/templates/_helpers.tpl @@ -310,11 +310,6 @@ imagePullSecrets: mountPath: "/invoker-scripts/configureDNS.sh" subPath: "configureDNS.sh" {{- end }} -{{- if ne .Values.docker.registry.name "" }} - - name: scripts-dir - mountPath: "/invoker-scripts/dockerLogin.sh" - subPath: "dockerLogin.sh" -{{- end }} {{- end }} {{- end }} diff --git a/helm/openwhisk/templates/invoker-cm.yaml b/helm/openwhisk/templates/invoker-cm.yaml index 0c689a31..a09f5075 100644 --- a/helm/openwhisk/templates/invoker-cm.yaml +++ b/helm/openwhisk/templates/invoker-cm.yaml @@ -24,4 +24,3 @@ metadata: data: {{ (.Files.Glob "configMapFiles/dockerPullRuntimes/playbook.yml").AsConfig | indent 2 }} {{ (.Files.Glob "configMapFiles/configureDNS/configureDNS.sh").AsConfig | indent 2 }} -{{ (.Files.Glob "configMapFiles/dockerLogin/dockerLogin.sh").AsConfig | indent 2 }} diff --git a/helm/openwhisk/templates/invoker-pod.yaml b/helm/openwhisk/templates/invoker-pod.yaml index e00ae3a3..42943249 100644 --- a/helm/openwhisk/templates/invoker-pod.yaml +++ b/helm/openwhisk/templates/invoker-pod.yaml @@ -89,9 +89,6 @@ spec: - /bin/bash - -c - -{{- if ne .Values.docker.registry.name "" }} - . /invoker-scripts/dockerLogin.sh && -{{- end }} {{- if and (eq .Values.invoker.containerFactory.impl "docker") .Values.invoker.containerFactory.networkConfig.dns.inheritInvokerConfig }} . /invoker-scripts/configureDNS.sh && {{- end }} diff --git a/helm/openwhisk/templates/seccomp-pod.yaml b/helm/openwhisk/templates/seccomp-pod.yaml index 6c897e1f..8a4b6626 100644 --- a/helm/openwhisk/templates/seccomp-pod.yaml +++ b/helm/openwhisk/templates/seccomp-pod.yaml @@ -35,10 +35,3 @@ spec: - name: profiles configMap: name: {{ .Release.Name }}-seccomp-profile -{{- if .Values.affinity.enabled }} - affinity: -{{ include "openwhisk.affinity.invoker" . | indent 8 }} -{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }} -{{ include "openwhisk.affinity.selfAntiAffinity" ( printf "%s-invoker" .Release.Name ) | indent 8 }} -{{- end }} -{{- end }}