Skip to content

Commit 6136aa1

Browse files
justinsbjustinsb
committed
tests: add smoketest using kind
The start of an e2e smoketest that deploys into a cluster with the operator. A few (bigger) todos: * We should figure out how to capture the GCP traffic * We should ensure that we are running the images we actually push
1 parent 25a7af3 commit 6136aa1

File tree

3 files changed

+144
-93
lines changed

3 files changed

+144
-93
lines changed

‎.github/workflows/ci-presubmit.yaml

Lines changed: 6 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -33,61 +33,27 @@ on:
3333

3434

3535
jobs:
36-
tests-scenarios:
37-
runs-on: ubuntu-latest
38-
timeout-minutes: 60
39-
steps:
40-
- uses: actions/checkout@v4
41-
- uses: actions/setup-go@v5
42-
with:
43-
go-version-file: 'go.mod'
44-
- name: "Run scripts/github-actions/tests-scenarios"
45-
run: |
46-
./scripts/github-actions/tests-scenarios
47-
env:
48-
ARTIFACTS: /tmp/artifacts
49-
- name: "Upload artifacts"
50-
uses: actions/upload-artifact@v4
51-
with:
52-
name: artifacts-tests-scenarios
53-
path: /tmp/artifacts/
5436

55-
tests-scenarios-acquisition:
37+
smoketest-with-kind:
5638
runs-on: ubuntu-latest
5739
timeout-minutes: 60
5840
steps:
5941
- uses: actions/checkout@v4
6042
- uses: actions/setup-go@v5
6143
with:
6244
go-version-file: 'go.mod'
63-
- name: "Run scripts/github-actions/tests-scenarios-acquisition"
64-
run: |
65-
./scripts/github-actions/tests-scenarios-acquisition
66-
env:
67-
ARTIFACTS: /tmp/artifacts
68-
- name: "Upload artifacts"
69-
uses: actions/upload-artifact@v4
70-
with:
71-
name: artifacts-tests-scenarios-acquisition
72-
path: /tmp/artifacts/
73-
tests-scenarios-powertool:
74-
runs-on: ubuntu-latest
75-
timeout-minutes: 10
76-
steps:
77-
- uses: actions/checkout@v4
78-
- uses: actions/setup-go@v5
79-
with:
80-
go-version-file: 'go.mod'
81-
- name: "Run scripts/github-actions/tests-scenarios-powertool"
45+
- name: "Run dev/ci/presubmits/smoketest-with-kind"
8246
run: |
83-
./scripts/github-actions/tests-scenarios-powertool
47+
./dev/ci/presubmits/smoketest-with-kind
8448
env:
8549
ARTIFACTS: /tmp/artifacts
8650
- name: "Upload artifacts"
8751
uses: actions/upload-artifact@v4
8852
with:
89-
name: artifacts-tests-scenarios-powertool
53+
name: artifacts-smoketest-with-kind
9054
path: /tmp/artifacts/
55+
56+
9157
test-mockgcp:
9258
runs-on: ubuntu-latest
9359
timeout-minutes: 60
@@ -96,10 +62,6 @@ jobs:
9662
- uses: actions/setup-go@v5
9763
with:
9864
go-version-file: 'go.mod'
99-
- uses: 'google-github-actions/setup-gcloud@v2'
100-
with:
101-
version: '520.0.0'
102-
install_components: 'alpha,beta'
10365
- name: "Run dev/ci/presubmits/test-mockgcp"
10466
run: |
10567
./dev/ci/presubmits/test-mockgcp

‎dev/ci/presubmits/smoketest-with-kind

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2024 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o errexit
17+
set -o nounset
18+
set -o pipefail
19+
20+
REPO_ROOT="$(git rev-parse --show-toplevel)"
21+
cd ${REPO_ROOT}
22+
23+
# Pick a likely unique version tag
24+
export IMAGE_TAG=dev-$(date +%Y%m%dT%H%M%S)
25+
26+
# We will side load our images into kind
27+
export IMAGE_PREFIX=registry.kind/
28+
29+
kind create cluster --name smoketest
30+
31+
dev/tasks/build-images
32+
33+
kind load --name smoketest docker-image ${IMAGE_PREFIX}operator:${IMAGE_TAG}
34+
35+
dev/tasks/deploy-to-kube
36+
37+
# Create namespace
38+
NS=config-control
39+
echo "Creating namespace ${NS}"
40+
kubectl create ns ${NS} --dry-run=client -oyaml | kubectl apply --server-side -f -
41+
42+
# echo "Creating ConfigConnectorContext in namespace ${NS} (with fake google service account)"
43+
# cat <<EOF | kubectl apply --server-side -f -
44+
# apiVersion: core.cnrm.cloud.google.com/v1beta1
45+
# kind: ConfigConnectorContext
46+
# metadata:
47+
# # you can only have one ConfigConnectorContext per namespace
48+
# name: configconnectorcontext.core.cnrm.cloud.google.com
49+
# namespace: ${NS}
50+
# spec:
51+
# googleServiceAccount: "[email protected]"
52+
# stateIntoSpec: Absent
53+
# EOF
54+
55+
# echo "Waiting for KCC bound to namespace ${NS} to become ready"
56+
# We don't wait, because prom-to-sd is currently crashing
57+
#kubectl wait -n cnrm-system --for=condition=Ready -l cnrm.cloud.google.com/scoped-namespace=${NS} pod
58+
59+
echo "Creating StorageBucket in namespace ${NS}"
60+
cat <<EOF | kubectl apply -f -
61+
apiVersion: storage.cnrm.cloud.google.com/v1beta1
62+
kind: StorageBucket
63+
metadata:
64+
name: "kcc-test-${NS}"
65+
namespace: "${NS}"
66+
annotations:
67+
cnrm.cloud.google.com/project-id: "test-project-1"
68+
spec:
69+
lifecycleRule:
70+
- action:
71+
type: Delete
72+
condition:
73+
age: 7
74+
withState: ANY
75+
versioning:
76+
enabled: true
77+
uniformBucketLevelAccess: true
78+
EOF
79+
80+
echo "Waiting for StorageBucket reconciliation"
81+
# Wait for StorageBucket creation attempt
82+
# We can't kubectl wait for ready, because we currently expect this to fail because we haven't set up IAM permissions
83+
# Instead we wait for the condition of _type_ Ready, but don't wait for status=True
84+
kubectl wait storagebucket -n ${NS} kcc-test-config-control --for=jsonpath='.status.conditions[].type=Ready'
85+
86+
87+
kubectl describe storagebucket -n ${NS}

‎dev/tasks/deploy-to-kube

Lines changed: 51 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -22,74 +22,76 @@ set -o pipefail
2222
REPO_ROOT="$(git rev-parse --show-toplevel)"
2323
cd ${REPO_ROOT}
2424

25-
if [[ -z "${VERSION:-}" ]]; then
26-
echo "VERSION must be set"
25+
if [[ -z "${IMAGE_TAG:-}" ]]; then
26+
echo "IMAGE_TAG must be set"
2727
exit 1
2828
fi
2929

3030
kustomize build operator/config/default | \
31-
sed -e "s@image: operator:.*@image: gcr.io/gke-release/cnrm/operator:${VERSION}@g" | \
31+
sed -e "s@image: operator:.*@image: gcr.io/gke-release/cnrm/operator:${IMAGE_TAG}@g" | \
32+
sed -e "s@imagePullPolicy: Always@imagePullPolicy: IfNotPresent@g" | \
3233
kubectl apply --server-side -n configconnector-operator-system -f -
3334

3435
echo "Waiting for configconnector-operator statefulset to become ready"
3536
kubectl wait -n configconnector-operator-system --for=jsonpath='{.status.readyReplicas}'=1 statefulset/configconnector-operator
3637

37-
# Configure in namespace mode, per instructions at https://cloud.google.com/config-connector/docs/how-to/install-namespaced
38-
echo "Configuring namespace mode"
38+
# # Configure in namespace mode, per instructions at https://cloud.google.com/config-connector/docs/how-to/install-namespaced
39+
# echo "Configuring namespace mode"
40+
# cat <<EOF | kubectl apply --server-side -f -
41+
# apiVersion: core.cnrm.cloud.google.com/v1beta1
42+
# kind: ConfigConnector
43+
# metadata:
44+
# # the name is restricted to ensure that there is only ConfigConnector resource installed in your cluster
45+
# name: configconnector.core.cnrm.cloud.google.com
46+
# spec:
47+
# mode: namespaced
48+
# stateIntoSpec: Absent
49+
# credentialSecretName: kcc-google-service-account
50+
# EOF
51+
52+
# Configure in cluster mode so we can use a (fake) google service account
53+
echo "Creating namespace cnrm-system"
54+
kubectl create ns cnrm-system --dry-run=client -oyaml | kubectl apply --server-side -f -
55+
56+
echo "Configuring ConfigConnector in cluster mode"
3957
cat <<EOF | kubectl apply --server-side -f -
58+
apiVersion: v1
59+
kind: Secret
60+
metadata:
61+
name: kcc-google-service-account
62+
namespace: cnrm-system
63+
type: Opaque
64+
stringData:
65+
key.json: |
66+
{
67+
"type": "service_account",
68+
"project_id": "fake-project-id",
69+
"private_key_id": "fake-private-key-id",
70+
"private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAsGHDAdHZfi81LgVeeMHXYLgNDpcFYhoBykYtTDdNyA5AixID\n8JdKlCmZ6qLNnZrbs4JlBJfmzw6rjUC5bVBFg5NwYVBu3+3Msa4rgLsTGsjPH9rt\nC+QFnFhcmzg3zz8eeXBqJdhw7wmn1Xa9SsC3h6YWveBk98ecyE7yGe8J8xGphjk7\nEQ/KBmRK/EJD0ZwuYW1W4Bv5f5fca7qvi9rCprEmL8//uy0qCwoJj2jU3zc5p72M\npkSZb1XlYxxTEo/h9WCEvWS9pGhy6fJ0sA2RsBHqU4Y5O7MJEei9yu5fVSZUi05f\n/ggfUID+cFEq0Z/A98whKPEBBJ/STdEaqEEkBwIDAQABAoIBAED6EsvF0dihbXbh\ntXbI+h4AT5cTXYFRUV2B0sgkC3xqe65/2YG1Sl0gojoE9bhcxxjvLWWuy/F1Vw93\nS5gQnTsmgpzm86F8yg6euhn3UMdqOJtknDToMITzLFJmOHEZsJFOL1x3ysrUhMan\nsn4qVrIbJn+WfbumBoToSFnzbHflacOh06ZRbYa2bpSPMfGGFtwqQjRadn5+pync\nlCjaupcg209sM0qEk/BDSzHvWL1VgLMdiKBx574TSwS0o569+7vPNt92Ydi7kARo\nreOzkkF4L3xNhKZnmls2eGH6A8cp1KZXoMLFuO+IwvBMA0O29LsUlKJU4PjBrf+7\nwaslnMECgYEA5bJv0L6DKZQD3RCBLue4/mDg0GHZqAhJBS6IcaXeaWeH6PgGZggV\nMGkWnULltJIYFwtaueTfjWqciAeocKx+rqoRjuDMOGgcrEf6Y+b5AqF+IjQM66Ll\nIYPUt3FCIc69z5LNEtyP4DSWsFPJ5UhAoG4QRlDTqT5q0gKHFjeLdeECgYEAxJRk\nkrsWmdmUs5NH9pyhTdEDIc59EuJ8iOqOLzU8xUw6/s2GSClopEFJeeEoIWhLuPY3\nX3bFt4ppl/ksLh05thRs4wXRxqhnokjD3IcGu3l6Gb5QZTYwb0VfN+q2tWVEE8Qc\nPQURheUsM2aP/gpJVQvNsWVmkT0Ijc3J8bR2hucCgYEAjOF4e0ueHu5NwFTTJvWx\nHTRGLwkU+l66ipcT0MCvPW7miRk2s3XZqSuLV0Ekqi/A3sF0D/g0tQPipfwsb48c\n0/wzcLKoDyCsFW7AQG315IswVcIe+peaeYfl++1XZmzrNlkPtrXY+ObIVbXOavZ5\nzOw0xyvj5jYGRnCOci33N4ECgYA91EKx2ABq0YGw3aEj0u31MMlgZ7b1KqFq2wNv\nm7oKgEiJ/hC/P673AsXefNAHeetfOKn/77aOXQ2LTEb2FiEhwNjiquDpL+ywoVxh\nT2LxsmqSEEbvHpUrWlFxn/Rpp3k7ElKjaqWxTHyTii2+BHQ+OKEwq6kQA3deSpy6\n1jz1fwKBgQDLqbdq5FA63PWqApfNVykXukg9MASIcg/0fjADFaHTPDvJjhFutxRP\nppI5Q95P12CQ/eRBZKJnRlkhkL8tfPaWPzzOpCTjID7avRhx2oLmstmYuXx0HluE\ncqXLbAV9WDpIJ3Bpa/S8tWujWhLDmixn2JeAdurWS+naH9U9e4I6Rw==\n-----END RSA PRIVATE KEY-----\n",
71+
"client_email": "[email protected]",
72+
"client_id": "fake-client-id",
73+
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
74+
"token_uri": "https://oauth2.googleapis.com/token",
75+
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
76+
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/[email protected]"
77+
}
78+
---
79+
4080
apiVersion: core.cnrm.cloud.google.com/v1beta1
4181
kind: ConfigConnector
4282
metadata:
4383
# the name is restricted to ensure that there is only ConfigConnector resource installed in your cluster
4484
name: configconnector.core.cnrm.cloud.google.com
4585
spec:
46-
mode: namespaced
86+
mode: cluster
4787
stateIntoSpec: Absent
88+
credentialSecretName: kcc-google-service-account
4889
EOF
4990

50-
# Create namespace
51-
NS=config-control
52-
echo "Creating namespace ${NS}"
53-
kubectl create ns ${NS} --dry-run=client -oyaml | kubectl apply --server-side -f -
54-
55-
echo "Creating ConfigConnectorContext in namespace ${NS} (with fake google service account)"
56-
cat <<EOF | kubectl apply --server-side -f -
57-
apiVersion: core.cnrm.cloud.google.com/v1beta1
58-
kind: ConfigConnectorContext
59-
metadata:
60-
# you can only have one ConfigConnectorContext per namespace
61-
name: configconnectorcontext.core.cnrm.cloud.google.com
62-
namespace: ${NS}
63-
spec:
64-
googleServiceAccount: "[email protected]"
65-
stateIntoSpec: Absent
66-
EOF
6791

68-
echo "Waiting for KCC bound to namespace ${NS} to become ready"
69-
# We don't wait, because prom-to-sd is currently crashing
70-
#kubectl wait -n cnrm-system --for=condition=Ready -l cnrm.cloud.google.com/scoped-namespace=${NS} pod
7192

72-
echo "Creating StorageBucket in namespace ${NS}"
73-
cat <<EOF | kubectl apply -f -
74-
apiVersion: storage.cnrm.cloud.google.com/v1beta1
75-
kind: StorageBucket
76-
metadata:
77-
name: "kcc-test-${NS}"
78-
namespace: "${NS}"
79-
spec:
80-
lifecycleRule:
81-
- action:
82-
type: Delete
83-
condition:
84-
age: 7
85-
withState: ANY
86-
versioning:
87-
enabled: true
88-
uniformBucketLevelAccess: true
89-
EOF
93+
echo "Waiting for StorageBucket CRD to be created"
94+
kubectl wait --for=create crd/storagebuckets.storage.cnrm.cloud.google.com
9095

91-
# Wait for StorageBucket creation attempt
92-
# We can't kubectl wait, because we currently expect this to fail because we haven't set up IAM permissions
93-
echo "Sleeping to allow for attempt at StorageBucket creation"
94-
sleep 5
95-
kubectl describe storagebucket -n ${NS}
96+
echo "Waiting for StorageBucket CRD to become ready"
97+
kubectl wait --for=jsonpath='{.status.acceptedNames.kind}'=StorageBucket crd/storagebuckets.storage.cnrm.cloud.google.com

0 commit comments

Comments
 (0)