diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 0b184311..137d119a 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -5,12 +5,13 @@ *** xref:serverless:service-mesh/common-service-mesh-network-isolation.adoc[Use Service Mesh to isolate network-traffic] *** xref:serverless:service-mesh/eventing-service-mesh-containersource.adoc[Eventing: Using ContainerSource with OpenShift Service Mesh] *** xref:serverless:service-mesh/eventing-service-mesh-sinkbinding.adoc[Eventing: Using SinkBinding with OpenShift Service Mesh] -** Features -*** xref:serverless:features/transport-encryption/transport-encryption-setup.adoc[Setup transport encryption in Eventing] -*** xref:serverless:features/eventing-kafka-scaling/eventing-kafka-scaling-setup.adoc[Setup Autoscaling For Eventing Kafka Components] +** Eventing +*** xref:serverless:eventing/transport-encryption-setup.adoc[Setup transport encryption in Eventing] +*** xref:serverless:eventing/kafka-scaling-setup.adoc[Setup Autoscaling For Eventing Kafka Components] ** Serving *** xref:serverless:serving/serving-with-ingress-sharding.adoc[Use Serving with OpenShift ingress sharding] *** xref:serverless:serving/scaleability-and-performance-of-serving.adoc[Scalability and performance of {serverlessproductname} Serving] +*** xref:serverless:serving/serving-transport-encryption.adoc[Serving Transport Encryption] * Serverless Logic ** xref:serverless-logic:about.adoc[About OpenShift Serverless Logic] ** xref:serverless-logic:release-notes.adoc[Release notes] diff --git a/modules/serverless/assets/images/serving-encryption/encryption-cluster-local-domain.drawio.svg b/modules/serverless/assets/images/serving-encryption/encryption-cluster-local-domain.drawio.svg new file mode 100644 index 00000000..32e8820e --- /dev/null +++ b/modules/serverless/assets/images/serving-encryption/encryption-cluster-local-domain.drawio.svg @@ -0,0 +1,4 @@ + + + +



































Cluster
Cluster...





Namespace: cert-manager
Namespace: cert-manager...






Namespace of ingress-controller
Namespace of ingress-controller...




















Namespace: user-2
Namespace: user-2...





Namespace: knative-serving
Namespace: knative-serving...
Kservice
with Queue-Proxy
Kservice...
Ingress Controller
(Kourier, Istio)
Ingress Controller...
trusts the CA
trusts the CA
create
create
Serving Controller
Serving Controller
Get signed
certificate
Get signed...
populate
populate
cert-manager
cert-manager
read
read
Knative Certificate
visibility: cluster-local
dnsName: myapp.user-2.svc.cluster.local
Knative Certificate...
read
read
Secret
name: <id>
Secret...
Uses SNI to return the matching certificate
Uses SNI to r...
Certificate Authority
(e.g. Company CA, Vault, allowed to sign *.cluster.local)
Certificate Authority...
Namespace: user-1






Namespace: user-1...
User application
(Knative or non-Knative workload)
User application...
myapp.user-2.svc.cluster.local
myapp.user-2.svc.cluster.local
Knative Component
Knative Component
Kubernetes Secret
Kubernetes Secret
Knative Certificate CR
Knative Certificate CR
Traffic flow
Traffic flow
Text is not SVG - cannot display
\ No newline at end of file diff --git a/modules/serverless/assets/images/serving-encryption/encryption-overview.drawio.svg b/modules/serverless/assets/images/serving-encryption/encryption-overview.drawio.svg new file mode 100644 index 00000000..430bbe8b --- /dev/null +++ b/modules/serverless/assets/images/serving-encryption/encryption-overview.drawio.svg @@ -0,0 +1,4 @@ + + + +


















Cluster
Cluster...
Namespace of ingress-controller






Namespace of ingress-controller...






Namespace: user-2
Namespace: user-2...






Namespace: knative-serving
Namespace: knative-serving...
(3)
(3)
Activator
Activator
Kservice
with Queue-Proxy
Kservice...
(3)
(3)
Ingress Controller
(Kourier, Istio)
Ingress Controller...
(1)
(1)
Application/User
external to the cluster
Applic...
Namespace: user-1






Namespace: user-1...
User application
(Knative or non-Knative workload)
User application...
(2)
(2)
(3)
(3)
Legend
Legend
(1) = external domains like myapp-user-2.example.com
(2) = cluster-local domain like myapp.user-2.svc.cluster.local
(3) = Knative Internal communication, transparent to the user
(1) = external domains like myapp-user-2.example.com...
Text is not SVG - cannot display
\ No newline at end of file diff --git a/modules/serverless/assets/images/serving-encryption/encryption-system-internal.drawio.svg b/modules/serverless/assets/images/serving-encryption/encryption-system-internal.drawio.svg new file mode 100644 index 00000000..b44b7f35 --- /dev/null +++ b/modules/serverless/assets/images/serving-encryption/encryption-system-internal.drawio.svg @@ -0,0 +1,4 @@ + + + +

































Cluster
Cluster...






Namespace: cert-manager
Namespace: cert-manager...
Namespace of ingress-controller











Namespace of ingress-controller...












Namespace: user-1
Namespace: user-1...











Namespace: knative-serving
Namespace: knative-serving...
Activator
Activator
Kservice
with Queue-Proxy
Kservice...
Ingress Controller
(Kourier, Istio)
Ingress Controller...
populate
populate
Secret
name: serving-certs
Secret...
use
use
Secret
name: routing-serving-certs
Secret...
populate
populate
use
use
trust
trust
Knative Component
Knative Component
Kubernetes Secret
Kubernetes Secret
Traffic flow
Traffic flow
cert-manager
cert-manager
Certificate Authority
(e.g. Company CA, Vault, allowed to sign defined SANs)
Certificate Authority...
Get signed
 certificate
Get signed...
Secret
name: routing-serving-certs
Secret...
use
use
Text is not SVG - cannot display
\ No newline at end of file diff --git a/modules/serverless/pages/features/eventing-kafka-scaling/eventing-kafka-scaling-setup.adoc b/modules/serverless/pages/eventing/kafka-scaling-setup.adoc similarity index 100% rename from modules/serverless/pages/features/eventing-kafka-scaling/eventing-kafka-scaling-setup.adoc rename to modules/serverless/pages/eventing/kafka-scaling-setup.adoc diff --git a/modules/serverless/pages/features/transport-encryption/transport-encryption-setup.adoc b/modules/serverless/pages/eventing/transport-encryption-setup.adoc similarity index 100% rename from modules/serverless/pages/features/transport-encryption/transport-encryption-setup.adoc rename to modules/serverless/pages/eventing/transport-encryption-setup.adoc diff --git a/modules/serverless/pages/serving/serving-transport-encryption.adoc b/modules/serverless/pages/serving/serving-transport-encryption.adoc new file mode 100644 index 00000000..5631956e --- /dev/null +++ b/modules/serverless/pages/serving/serving-transport-encryption.adoc @@ -0,0 +1,450 @@ += {serverlessproductname} Serving Transport Encryption +:compat-mode!: +// Metadata: +:description: {serverlessproductname} Serving Transport Encryption +:certmanagerlink: https://docs.redhat.com/en/documentation/openshift_container_platform/4.15/html/security_and_compliance/cert-manager-operator-for-red-hat-openshift#installing-the-cert-manager-operator-for-red-hat-openshift + +{serverlessproductname} Serving transport encryption feature allows transporting data over secured and encrypted HTTPS connections using TLS. + +[IMPORTANT] +==== +{serverlessproductname} transport encryption for Serving is a Developer Preview feature only. +Developer Preview features are not supported with Red Hat production service level agreements (SLAs) and might not be functionally complete. +Red Hat does not recommend using them in production. +These features provide early access to upcoming product features, enabling customers to test functionality and provide feedback during the development process. + +For more information about the support scope of Red Hat Developer Preview features, see https://access.redhat.com/support/offerings/devpreview/. +==== + + + +== Overview + +There are three parts to {serverlessproductname} Serving Transport Encryption: + +image::serving-encryption/encryption-overview.drawio.svg[] + +1. <>: Transport Encryption on the ingress layer external to the cluster (e.g. cluster external domain, like `myapp-.example.com`). +2. <>: Transport Encryption on the ingress layer internal to the cluster (e.g. cluster local domains, like `myapp..svc.cluster.local`). +3. <>: Transport Encryption between Knative internal components (ingress-gateway, activator, queue-proxy). + +[IMPORTANT] +==== +Currently, all control-plane traffic (including Kubernetes PreStopHooks, metadata and metrics) is not (yet) encrypted. +As this contains no user data, it is considered to be implemented in a future release. +==== + +The different parts are independent of each other and can be enabled/disabled individually. Also, they can use the same or different Certificate Authorities (CAs) to sign the necessary certificates. + +[[external-domain-encryption]] +=== External domain encryption + +The transport encryption for external domains is handled by the cluster's ingress layer. +This is either {product-title} ingress or {smproductname}. +Please refer to the relevant documentation for more information. + +[[cluster-local-encryption]] +=== Cluster-local encryption + +image::serving-encryption/encryption-cluster-local-domain.drawio.svg[] + +This part enables transport encryption for cluster-local domains. It has the following properties: + +* Certificate CN/Subject Alternative Name (SAN) contains the cluster-local domains of a Knative Service, e.g. `myapp.namespace.svc.cluster.local`, `myapp.namespace.svc`, `myapp.namespace`. +* The certificates are selected using SNI by the cluster-local endpoint of the ingress-controller. +* The caller has to trust the CA that signed the cluster-local certificates (this is out of the scope of {ServerlessProductName}). +* To create the certificates, Knative relies on cert-manager. {certmanagerlink}[cert-manager needs to be installed] and configured for the feature to work. + +[[system-internal-encryption]] +=== {ServerlessProductName} Serving system-internal encryption + +image::serving-encryption/encryption-system-internal.drawio.svg[] + +This part enables transport encryption for system internal components (Ingress-Gateway, Activator, Queue-Proxy). These components are hosting TLS endpoints when this configuration is used: + +* To get the certificates, {ServerlessProductName} relies on cert-manager. {certmanagerlink}[cert-manager needs to be installed] and configured for the feature to work. +* Specific SANs are used to verify each connection. Each component needs to trust the CA that signed the certificates. For this, {ServerlessProductName} system components will consume and trust a bundle of CAs. The CA bundle needs to be provided by the cluster administrator. + + +== Selecting a certificate issuer [[issuer_selection]] + +[INFO] +==== +Issuers here are referring to cert-manager `Issuers` and `ClusterIssuers`. They represent certificate authorities (CAs) that can generate signed certificates by honoring certificate signing requests. +Reference: https://cert-manager.io/docs/concepts/issuer/ +==== + +For identifying your certificate issuer, you can refer to the cert-manager documentation (https://cert-manager.io/docs/configuration/issuers/). There are examples available for: + +* A custom CA stored in a K8s secret +* HTTP-01 challenges, e.g. Let's encrypt +* DNS-01 challenges +* Self-signed issuers + +[IMPORTANT] +==== +Please note, that not all issuer types work for each Knative Serving encryption feature. +==== + +Depending on the part you would like to use, {serverlessproductname} requires your certificate issuer to be capable of signing the following certificates: + +For cluster-local encryption, the issuer needs to be able to sign certificates for cluster-local domains like: + +* `myapp.` +* `myapp..svc` +* `myapp..svc.cluster.local` + +As the CA usually is not within the cluster, verification via the ACME protocol (DNS01/HTTP01) is not possible. You can use an issuer that allows creating these certificates (e.g. cert-manager's CA issuer). + +For system-internal encryption, the issuer must be able to sign certificates with the following SANs: + +* `kn-routing` +* `kn-user-` ( is each namespace where Knative Services are/will be created) +* `data-plane.knative.dev` + +These SANs are needed for Knative to verify connections between the internal components. +As this is also not possible via ACME protocol (DNS01/HTTP01), you need to configure an issuer that allows creating these certificates (e.g. cert-manager's CA issuer). + + +== Setting up {ServerlessProductName} transport encryption + +The setup is required for any of the transport encryption parts. + +=== Prerequisites + +* You have access to an {product-title} account with cluster administrator access. + +* Install the OpenShift CLI (`oc`). + +* {certmanagerlink}[Install the {certmanageroperatorname}]. + +* Install the {serverlessoperatorname}. + +[IMPORTANT] +==== +If you have installed the {serverlessoperatorname} before installing the {certmanageroperatorname}, you will have to restart the following components to enable the Knative Serving cert-manager integration. If this is not done, Knative will not create the necessary cert-manager resources, leading to pending Knative Services. + +* Controller deployment in the `knative-serving` namespace. +* Activator deployment in the `knative-serving` namespace. +==== + +=== Setup a `SelfSigned` `ClusterIssuer` [[setup_selfsigned_clusterissuer]] + +[IMPORTANT] +==== +For the simplicity of this guide, we will use a `SelfSigned` issuer as root certificate, however, be aware of the implications and limitations as documented at https://cert-manager.io/docs/configuration/selfsigned/ of this method. + +If you're running your company-specific Private Key Infrastructure (PKI), we recommend the CA issuer. +Refer to the cert-manager documentation for more details: https://cert-manager.io/docs/configuration/ca/, however, you can use any other issuer that allows signing of certificates as described above. +==== + +. Create a `SelfSigned` `ClusterIssuer`: ++ +[source,yaml] +---- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: knative-serving-selfsigned-issuer +spec: + selfSigned: {} +---- ++ +. Apply the `ClusterIssuer` resource: ++ +[source,terminal] +---- +$ oc apply -f +---- + +. Create a root certificate using the previously created `SelfSigned` `ClusterIssuer`: ++ +[source,yaml] +---- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: knative-serving-selfsigned-ca + namespace: cert-manager <1> +spec: + secretName: knative-serving-ca <2> + + isCA: true + commonName: selfsigned-ca + privateKey: + algorithm: ECDSA + size: 256 + + issuerRef: + name: knative-serving-selfsigned-issuer + kind: ClusterIssuer + group: cert-manager.io +---- ++ +<1> The {certmanageroperatorname} namespace, cert-manager by default. +<2> Secret name later used for the `ClusterIssuer` for Serving ++ +. Apply the `Certificate` resource: ++ +[source,terminal] +---- +$ oc apply -f +---- + +=== Creating a `ClusterIssuer` to be used by Serving + +. Create the `knative-serving-ca-issuer` `ClusterIssuer` for Serving: ++ +[source,yaml] +---- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: knative-serving-ca-issuer +spec: + ca: + secretName: knative-serving-ca <1> +---- ++ +<1> Secret name in the {certmanageroperatorname} namespace (cert-manager by default) containing the certificate that can then be used by {serverlessproductname} Serving components for new certificates. ++ +. Apply the `ClusterIssuer` resource: ++ +[source,terminal] +---- +$ oc apply -f +---- + +=== Understanding and configuring the transport encryption configuration + +. The transport encryption configuration consists of two configurations: ++ +The configuration of which `ClusterIssuer` to use: + +* `clusterLocalIssuerRef`: issuer for cluster-local-domain certificates used for ingress. +* `systemInternalIssuerRef`: issuer for certificates for system-internal-tls certificates used by Knative internal components. + ++ +The configuration on which transport encryption features to use: + +* `cluster-local-domain-tls`: Enables the transport encryption feature for cluster-local domains +* `system-internal-tls`: Enables the transport encryption feature for {serverlessproductname} Serving internal components. + + +. Enabling transport-encryption in `KnativeServing`: ++ +[source,yaml] +---- +apiVersion: operator.knative.dev/v1beta1 +kind: KnativeServing +metadata: + name: knative-serving + namespace: knative-serving +spec: + # Other spec fields omitted ... + config: + certmanager: + clusterLocalIssuerRef: | + kind: ClusterIssuer + name: knative-serving-ca-issuer <1> + systemInternalIssuerRef: | + kind: ClusterIssuer + name: knative-serving-ca-issuer <1> + network: + cluster-local-domain-tls: Enabled <2> + system-internal-tls: Enabled <3> +---- ++ +<1> Define the `ClusterIssuer` for each feature. The same or individual `ClusterIssuers` can be used. +<2> Enabling the `cluster-local-domain-tls` feature. They can be enabled/disabled individually. +<3> Enabling the `system-internal-tls` feature. They can be enabled/disabled individually. + +. Apply the `KnativeServing` resource: ++ +[source,terminal] +---- +$ oc apply -f +---- + +. Restart the Controller component if you enabled `cluster-local-domain-tls` or `system-internal-tls`: ++ +[IMPORTANT] +==== +When either the `cluster-local-domain-tls` or the `system-internal-tls` feature is enabled, the Controller component needs to be restarted to enable the Knative Serving cert-manager integration. +==== ++ +[source,terminal] +---- +$ oc rollout restart deploy/controller -n knative-serving +---- + +. Restart the Activator component if you enabled `system-internal-tls` ++ +[IMPORTANT] +==== +When the `system-internal-tls` feature is activated, the Activator component needs to be restarted to reconfigure its internal web server, as this is not possible during runtime. +==== ++ +[source,terminal] +---- +$ oc rollout restart deploy/activator -n knative-serving +---- + + +== Configure trust + +When you enable any of the transport encryption features, you must make sure that all clients calling do trust the Certificate Authority (CA) that issues the certificates used for the transport encryption. + +There are multiple places where trust needs to be ensured: + +* Cluster external client (Browser and/or other application): this is considered out of the scope of {serverlessproductname}. +* {serverlessproductname} system components (e.g. Activator, Queue-Proxy, Ingress-Controller): see below. +* Cluster internal client (e.g. a Knative Service or other workload): see below. + +=== Configuring trust for {serverlessproductname} Serving components and Knative Services [[configuring_trust]] + +For {serverlessproductname} Serving components and Knative Services to trust the CA that issues certificates, you can create a `ConfigMap` in the following namespaces with the label `networking.knative.dev/trust-bundle: true`: + +* `knative-serving`: for the system components of {serverlessproductname} Serving. +* `knative-serving-ingress`: for the ingress layer of {serverlessproductname} Serving. +* `istio-system` or your own {smproductshortname} namespace: when the {smproductshortname} integration is enabled. + +Knative looks for ConfigMaps with this label and will read all data keys (regardless of the name). +One key can contain one or multiple CAs/Intermediates. If they are valid, they will be added to the trust store of the Knative components. + +Here is an example of how ConfigMap could look like: +[source,yaml] +---- +apiVersion: v1 +data: + cacerts.pem: | <1> + -----BEGIN CERTIFICATE----- + MIIDDTCCAfWgAwIBAgIQMQuip05h7NLQq2TB+j9ZmTANBgkqhkiG9w0BAQsFADAW + MRQwEgYDVQQDEwtrbmF0aXZlLmRldjAeFw0yMzExMjIwOTAwNDhaFw0yNDAyMjAw + OTAwNDhaMBYxFDASBgNVBAMTC2tuYXRpdmUuZGV2MIIBIjANBgkqhkiG9w0BAQEF + AAOCAQ8AMIIBCgKCAQEA3clC3CV7sy0TpUKNuTku6QmP9z8JUCbLCPCLACCUc1zG + FEokqOva6TakgvAntXLkB3TEsbdCJlNm6qFbbko6DBfX6rEggqZs40x3/T+KH66u + 4PvMT3fzEtaMJDK/KQOBIvVHrKmPkvccUYK/qWY7rgBjVjjLVSJrCn4dKaEZ2JNr + Fd0KNnaaW/dP9/FvviLqVJvHnTMHH5qyRRr1kUGTrc8njRKwpHcnUdauiDoWRKxo + Zlyy+MhQfdbbyapX984WsDjCvrDXzkdGgbRNAf+erl6yUm6pHpQhyFFo/zndx6Uq + QXA7jYvM2M3qCnXmaFowidoLDsDyhwoxD7WT8zur/QIDAQABo1cwVTAOBgNVHQ8B + Af8EBAMCAgQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zAd + BgNVHQ4EFgQU7p4VuECNOcnrP9ulOjc4J37Q2VUwDQYJKoZIhvcNAQELBQADggEB + AAv26Vnk+ptQrppouF7yHV8fZbfnehpm07HIZkmnXO2vAP+MZJDNrHjy8JAVzXjt + +OlzqAL0cRQLsUptB0btoJuw23eq8RXgJo05OLOPQ2iGNbAATQh2kLwBWd/CMg+V + KJ4EIEpF4dmwOohsNR6xa/JoArIYH0D7gh2CwjrdGZr/tq1eMSL+uZcuX5OiE44A + 2oXF9/jsqerOcH7QUMejSnB8N7X0LmUvH4jAesQgr7jo1JTOBs7GF6wb+U76NzFa + 8ms2iAWhoplQ+EHR52wffWb0k6trXspq4O6v/J+nq9Ky3vC36so+G1ZFkMhCdTVJ + ZmrBsSMWeT2l07qeei2UFRU= + -----END CERTIFICATE----- +kind: ConfigMap +metadata: + labels: + networking.knative.dev/trust-bundle: "true" + name: knative-bundle <2> + namespace: knative-serving +---- +<1> All keys containing valid PEM-encoded CA bundles will be trusted by Serving components. +<2> You can define your own name. + +[IMPORTANT] +==== +Whenever a CA bundle `ConfigMap` is created or updated, the Serving components will automatically pick them up and add the CAs/Intermediate certificates to their CA trust store. The trust store is refreshed for every new HTTP connection. +==== + +=== Configuring trust on your custom workload [[trust_custom_workload]] + +As {serverlessproductname} Serving does not control all workloads and managing trust is highly dependent on your runtime and/or language, this area is out of the scope of {serverlessproductname}. But here are few options for how this could be achieved: + +* Adding the CA bundle to a Container image on build-time (be aware that this complicates CA rotation, you will need to rebuild and redeploy every application when the CA rotates). +* Mounting a CA bundle to the filesystem (e.g. from a `Secret` or `ConfigMap`) and making sure your application uses it to verify TLS connections. +* Reading it from environment variable and making sure your application uses it to verify TLS connections. +* Accessing it from a `Secret`/`ConfigMap` via K8s API and making sure your application uses it to verify TLS connections. + + +== Ensure seamless CA rotation + +Ensuring seamless CA rotation is essential to avoid service downtime, or to deal with an emergency. +The following procedure explains how you can seamlessly rotate a CA: + +1. Create a new CA certificate. + +2. Add the public key of the new CA certificate to the CA trust bundles as described in the <> section. Make sure to also keep the public key of the existing CA. + +3. Ensure that all clients have consumed the latest set of CA trust bundles. {serverlessproductname} Serving components will automatically reload the changed CA trust bundles. + +4. If you have custom workload consuming trust bundles as well, make sure to reload/restart them accordingly. + +5. Update the `knative-serving-ca-issuer` `ClusterIssuer` to reference the secret containing the CA certificate created in step 1. + +6. Either wait for cert-manager to renew all your certificates or enforce it to renew all the certificates. Refer to the cert-manager documentation for more details: https://cert-manager.io/docs/usage/certificate/#reissuance-triggered-by-user-actions. + +7. As soon as the CA rotation is fully completed (add some grace period to this to make sure all components did pick up the changes), you can remove the public key of the old CA from the trust bundle `ConfigMap`. + + +== Verification + +. Create a `KnativeService`: ++ +[source,yaml] +---- +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: test-webapp + namespace: test-namespace +spec: + template: + spec: + containers: + - image: docker.io/openshift/hello-openshift + env: + - name: RESPONSE + value: "Hello Serverless!" +---- +. Apply the `KnativeService` YAML: ++ +[source,terminal] +---- +$ oc apply -f +---- + +. Examine the `KnativeService` status: ++ +[source,terminal] +---- +$ oc get ksvc -n test-namespace -o yaml +---- ++ +.Example output +[source,yaml] +---- +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: test-webapp + namespace: test-namespace +# spec: +# ... +status: + address: + # cluster-local-domain: + url: https://helloworld.test.svc.cluster.local <1> +---- ++ +<1> If you have enabled `cluster-local-domain-tls` you will now see HTTPS url. + +. To verify if `system-internal-tls` is enabled, you can check the output of `Queue-Proxy` logs: ++ +[source,terminal] +---- +$ oc logs your-pod -n test-namespace -c queue-proxy | grep -E 'certDir|Certificate|tls' +---- +. Check the log output and look for lines similar to these: ++ +[source,terminal] +---- +{"severity":"INFO","timestamp":"2024-01-03T07:07:32.892810888Z","logger":"queueproxy","caller":"certificate/watcher.go:62","message":"Starting to watch the following directories for changes{certDir 15 0 /var/lib/knative/certs } {keyDir 15 0 /var/lib/knative/certs }","commit":"86420f2-dirty","knative.dev/key":"first/helloworld-00001","knative.dev/pod":"helloworld-00001-deployment-75fbb7d488-qgmxx"} +{"severity":"INFO","timestamp":"2024-01-03T07:07:32.89397512Z","logger":"queueproxy","caller":"certificate/watcher.go:131","message":"Certificate and/or key have changed on disk and were reloaded.","commit":"86420f2-dirty","knative.dev/key":"first/helloworld-00001","knative.dev/pod":"helloworld-00001-deployment-75fbb7d488-qgmxx"} +{"severity":"INFO","timestamp":"2024-01-03T07:07:32.894232939Z","logger":"queueproxy","caller":"sharedmain/main.go:282","message":"Starting tls server admin:8022","commit":"86420f2-dirty","knative.dev/key":"first/helloworld-00001","knative.dev/pod":"helloworld-00001-deployment-75fbb7d488-qgmxx"} +{"severity":"INFO","timestamp":"2024-01-03T07:07:32.894268548Z","logger":"queueproxy","caller":"sharedmain/main.go:282","message":"Starting tls server main:8112","commit":"86420f2-dirty","knative.dev/key":"first/helloworld-00001","knative.dev/pod":"helloworld-00001-deployment-75fbb7d488-qgmxx"} +----