From 6b2f2601fce0b0bc01e9a152a5982c7ad9568db8 Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Thu, 16 Dec 2021 08:51:29 +0100 Subject: [PATCH] Fix connection to vcluster (#6) * Fix connection to vcluster * Add in cluster names to serving cert --- component/cluster.libsonnet | 14 +++++-- component/post-setup.libsonnet | 8 +++- component/scripts/apply.sh | 7 +++- component/scripts/synthesize.sh | 7 +++- .../defaults/defaults/10_cluster.yaml | 8 ++++ tests/golden/oidc/oidc/oidc/10_cluster.yaml | 39 +++++++++++++------ .../openshift/openshift/10_cluster.yaml | 8 ++++ 7 files changed, 72 insertions(+), 19 deletions(-) diff --git a/component/cluster.libsonnet b/component/cluster.libsonnet index 76413a9..06fff3c 100644 --- a/component/cluster.libsonnet +++ b/component/cluster.libsonnet @@ -206,6 +206,12 @@ local cluster = function(name, options) emptyDir: {}, }, ], + local tlsSANs = [ + '--tls-san=%s.%s.svc.cluster.local' % [ name, options.namespace ], + '--tls-san=%s.%s.svc' % [ name, options.namespace ], + '--tls-san=%s.%s' % [ name, options.namespace ], + '--tls-san=%s' % [ name ], + ], containers: [ { image: formatImage(options.images.k3s), @@ -233,7 +239,7 @@ local cluster = function(name, options) '--flannel-backend=none', '--service-cidr=%s' % options.host_service_cidr, '--kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle', - ] + options.k3s.additional_args, + ] + tlsSANs + options.k3s.additional_args, env: [], securityContext: { allowPrivilegeEscalation: false, @@ -260,7 +266,7 @@ local cluster = function(name, options) args: [ '--name=' + name, '--out-kube-config-secret=vc-%s-kubeconfig' % name, - ] + options.syncer.additional_args, + ] + tlsSANs + options.syncer.additional_args, livenessProbe: { httpGet: { path: '/healthz', @@ -346,8 +352,8 @@ local cluster = function(name, options) headlessService, statefulSet, if options.ingress.host != null then ingress, - if std.length(options.additional_manifests) > 0 then postSetup.ApplyManifests('%s-apply-manifests' % name, 'vc-%s-kubeconfig' % name, options.additional_manifests), - if options.syn.registration_url != null then postSetup.Synthesize('%s-synthesize' % name, 'vc-%s-kubeconfig' % name, options.syn.registration_url), + if std.length(options.additional_manifests) > 0 then postSetup.ApplyManifests(name, 'vc-%s-kubeconfig' % name, options.additional_manifests), + if options.syn.registration_url != null then postSetup.Synthesize(name, 'vc-%s-kubeconfig' % name, options.syn.registration_url), ] + if options.ocp_route.host != null then ocpRoute.RouteCreateJob(name, 'vc-%s-kubeconfig' % name, options.ocp_route.host) else []); { diff --git a/component/post-setup.libsonnet b/component/post-setup.libsonnet index 175531c..3571b09 100644 --- a/component/post-setup.libsonnet +++ b/component/post-setup.libsonnet @@ -4,7 +4,8 @@ local kube = import 'lib/kube.libjsonnet'; local inv = kap.inventory(); local params = inv.parameters.vcluster; -local synthesize = function(jobName, secretName, url) +local synthesize = function(name, secretName, url) + local jobName = '%s-synthesize' % name; kube.Job(jobName) { metadata+: { namespace: params.namespace, @@ -23,6 +24,7 @@ local synthesize = function(jobName, secretName, url) args: [ '-eu', '-c', importstr './scripts/synthesize.sh', '--', url ], env: [ { name: 'HOME', value: '/export' }, + { name: 'VCLUSTER_SERVER_URL', value: 'https://%s:443' % name }, ], volumeMounts: [ { name: 'export', mountPath: '/export' }, @@ -39,7 +41,8 @@ local synthesize = function(jobName, secretName, url) }, }; -local applyManifests = function(jobName, secretName, manifests) +local applyManifests = function(name, secretName, manifests) + local jobName = '%s-apply-manifests' % name; local manifestArray = if std.isArray(manifests) then manifests else if std.isObject(manifests) then @@ -65,6 +68,7 @@ local applyManifests = function(jobName, secretName, manifests) args: [ '-eu', '-c', importstr './scripts/apply.sh', '--' ] + std.map(function(m) std.manifestJsonEx(m, ''), manifestArray), env: [ { name: 'HOME', value: '/export' }, + { name: 'VCLUSTER_SERVER_URL', value: 'https://%s:443' % name }, ], volumeMounts: [ { name: 'export', mountPath: '/export' }, diff --git a/component/scripts/apply.sh b/component/scripts/apply.sh index dd68d23..85305af 100755 --- a/component/scripts/apply.sh +++ b/component/scripts/apply.sh @@ -1,7 +1,12 @@ #!/bin/sh set -eu -vcluster_kubeconfig=/etc/vcluster-kubeconfig/config +cp /etc/vcluster-kubeconfig/config ./config +vcluster_kubeconfig=./config + +echo "Setting server URL..." + +kubectl --kubeconfig "$vcluster_kubeconfig" config set clusters.local.server "$VCLUSTER_SERVER_URL" echo "Applying manifests..." diff --git a/component/scripts/synthesize.sh b/component/scripts/synthesize.sh index f54d262..26e7aa8 100755 --- a/component/scripts/synthesize.sh +++ b/component/scripts/synthesize.sh @@ -1,7 +1,12 @@ #!/bin/sh set -eu -vcluster_kubeconfig=/etc/vcluster-kubeconfig/config +cp /etc/vcluster-kubeconfig/config ./config +vcluster_kubeconfig=./config + +echo "Setting server URL..." + +kubectl --kubeconfig "$vcluster_kubeconfig" config set clusters.local.server "$VCLUSTER_SERVER_URL" echo "Checking for namespace 'syn'..." diff --git a/tests/golden/defaults/defaults/defaults/10_cluster.yaml b/tests/golden/defaults/defaults/defaults/10_cluster.yaml index be91ef6..a28a0c0 100644 --- a/tests/golden/defaults/defaults/defaults/10_cluster.yaml +++ b/tests/golden/defaults/defaults/defaults/10_cluster.yaml @@ -158,6 +158,10 @@ spec: - --flannel-backend=none - --service-cidr=172.30.0.0/16 - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle + - --tls-san=defaults.syn-defaults.svc.cluster.local + - --tls-san=defaults.syn-defaults.svc + - --tls-san=defaults.syn-defaults + - --tls-san=defaults command: - /bin/k3s env: [] @@ -177,6 +181,10 @@ spec: - args: - --name=defaults - --out-kube-config-secret=vc-defaults-kubeconfig + - --tls-san=defaults.syn-defaults.svc.cluster.local + - --tls-san=defaults.syn-defaults.svc + - --tls-san=defaults.syn-defaults + - --tls-san=defaults env: [] image: docker.io/loftsh/vcluster:0.5.0-beta.0 livenessProbe: diff --git a/tests/golden/oidc/oidc/oidc/10_cluster.yaml b/tests/golden/oidc/oidc/oidc/10_cluster.yaml index f0a8dde..8095ed0 100644 --- a/tests/golden/oidc/oidc/oidc/10_cluster.yaml +++ b/tests/golden/oidc/oidc/oidc/10_cluster.yaml @@ -158,6 +158,10 @@ spec: - --flannel-backend=none - --service-cidr=172.30.0.0/16 - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle + - --tls-san=oidc.testns.svc.cluster.local + - --tls-san=oidc.testns.svc + - --tls-san=oidc.testns + - --tls-san=oidc - --kube-apiserver-arg=oidc-issuer-url=https://id.local/auth/realms/local - --kube-apiserver-arg=oidc-client-id=local - --kube-apiserver-arg=oidc-username-claim=email @@ -181,6 +185,10 @@ spec: - args: - --name=oidc - --out-kube-config-secret=vc-oidc-kubeconfig + - --tls-san=oidc.testns.svc.cluster.local + - --tls-san=oidc.testns.svc + - --tls-san=oidc.testns + - --tls-san=oidc env: [] image: docker.io/loftsh/vcluster:0.5.0-beta.0 livenessProbe: @@ -262,10 +270,13 @@ spec: - args: - -eu - -c - - "#!/bin/sh\nset -eu\n\nvcluster_kubeconfig=/etc/vcluster-kubeconfig/config\n\ - \necho \"Applying manifests...\"\n\nfor manifest in \"$@\"\ndo\n printf\ - \ \"$manifest\" | kubectl --kubeconfig \"$vcluster_kubeconfig\" apply\ - \ -f - -oyaml\ndone\n\necho \"Done!\"\n" + - "#!/bin/sh\nset -eu\n\ncp /etc/vcluster-kubeconfig/config ./config\n\ + vcluster_kubeconfig=./config\n\necho \"Setting server URL...\"\n\nkubectl\ + \ --kubeconfig \"$vcluster_kubeconfig\" config set clusters.local.server\ + \ \"$VCLUSTER_SERVER_URL\"\n\necho \"Applying manifests...\"\n\nfor\ + \ manifest in \"$@\"\ndo\n printf \"$manifest\" | kubectl --kubeconfig\ + \ \"$vcluster_kubeconfig\" apply -f - -oyaml\ndone\n\necho \"Done!\"\ + \n" - -- - '{ @@ -307,6 +318,8 @@ spec: env: - name: HOME value: /export + - name: VCLUSTER_SERVER_URL + value: https://oidc:443 image: quay.io/bitnami/kubectl:1.22.4 imagePullPolicy: IfNotPresent name: oidc-apply-manifests @@ -352,13 +365,15 @@ spec: - args: - -eu - -c - - "#!/bin/sh\nset -eu\n\nvcluster_kubeconfig=/etc/vcluster-kubeconfig/config\n\ - \necho \"Checking for namespace 'syn'...\"\n\nexists=$(kubectl --kubeconfig\ - \ \"$vcluster_kubeconfig\" get namespace syn --ignore-not-found)\nif\ - \ [ -n \"$exists\" ]; then\n echo \"Namespace 'syn' exists. Skipping\ - \ synthesize.\"\n exit 0\nfi\n\necho \"Starting synthesize...\"\n\n\ - kubectl --kubeconfig \"$vcluster_kubeconfig\" apply -f \"$1\"\n\necho\ - \ \"Done!\"\n" + - "#!/bin/sh\nset -eu\n\ncp /etc/vcluster-kubeconfig/config ./config\n\ + vcluster_kubeconfig=./config\n\necho \"Setting server URL...\"\n\nkubectl\ + \ --kubeconfig \"$vcluster_kubeconfig\" config set clusters.local.server\ + \ \"$VCLUSTER_SERVER_URL\"\n\necho \"Checking for namespace 'syn'...\"\ + \n\nexists=$(kubectl --kubeconfig \"$vcluster_kubeconfig\" get namespace\ + \ syn --ignore-not-found)\nif [ -n \"$exists\" ]; then\n echo \"Namespace\ + \ 'syn' exists. Skipping synthesize.\"\n exit 0\nfi\n\necho \"Starting\ + \ synthesize...\"\n\nkubectl --kubeconfig \"$vcluster_kubeconfig\" apply\ + \ -f \"$1\"\n\necho \"Done!\"\n" - -- - https://syn.example.com/steward/install.json?token=w84kxjbhf command: @@ -366,6 +381,8 @@ spec: env: - name: HOME value: /export + - name: VCLUSTER_SERVER_URL + value: https://oidc:443 image: quay.io/bitnami/kubectl:1.22.4 imagePullPolicy: IfNotPresent name: oidc-synthesize diff --git a/tests/golden/openshift/openshift/openshift/10_cluster.yaml b/tests/golden/openshift/openshift/openshift/10_cluster.yaml index 7193690..6c24280 100644 --- a/tests/golden/openshift/openshift/openshift/10_cluster.yaml +++ b/tests/golden/openshift/openshift/openshift/10_cluster.yaml @@ -164,6 +164,10 @@ spec: - --flannel-backend=none - --service-cidr=172.30.0.0/16 - --kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle + - --tls-san=openshift.syn-openshift.svc.cluster.local + - --tls-san=openshift.syn-openshift.svc + - --tls-san=openshift.syn-openshift + - --tls-san=openshift command: - /bin/k3s env: [] @@ -183,6 +187,10 @@ spec: - args: - --name=openshift - --out-kube-config-secret=vc-openshift-kubeconfig + - --tls-san=openshift.syn-openshift.svc.cluster.local + - --tls-san=openshift.syn-openshift.svc + - --tls-san=openshift.syn-openshift + - --tls-san=openshift env: [] image: docker.io/loftsh/vcluster:0.5.0-beta.0 livenessProbe: