Skip to content

Commit 0862a9a

Browse files
author
Akash Shrivastava
authored
Chore(Test): Addition of E2E test for Azure Experiments (#354)
* Added azure e2e tests Signed-off-by: Akash Shrivastava <[email protected]> * Added disk loss experiment into pipeline Signed-off-by: Akash Shrivastava <[email protected]> * Condensed az cli commands Shortened engine name Signed-off-by: Akash Shrivastava <[email protected]> * Changes in secret creation with azure credentials Signed-off-by: Akash Shrivastava <[email protected]> * Shortened engine name Signed-off-by: Akash Shrivastava <[email protected]> * fix: Changed engine name Signed-off-by: Akash Shrivastava <[email protected]> * Changed version to master for rbac file Signed-off-by: Akash Shrivastava <[email protected]> * Fixed issue with secret creation and instance stop test Signed-off-by: Akash Shrivastava <[email protected]> * Minor fix Signed-off-by: Akash Shrivastava <[email protected]> * Added run number to resource group name creation Signed-off-by: Akash Shrivastava <[email protected]> * Added disk loss e2e test Signed-off-by: Akash Shrivastava <[email protected]> * Added different resource group naming for multiple runs Signed-off-by: Akash Shrivastava <[email protected]> * Fixed issue in rg naming for disk loss test Signed-off-by: Akash Shrivastava <[email protected]> * Changed instance and disk from secrets Signed-off-by: Akash Shrivastava <[email protected]>
1 parent 2d6679f commit 0862a9a

9 files changed

+742
-2
lines changed

‎.github/workflows/azure-experiment-pipeline.yml

+203
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
---
2+
name: Azure-Experiment-Pipeline
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
goExperimentImage:
7+
description: "Go Experiment Image"
8+
default: "litmuschaos/go-runner:ci"
9+
required: true
10+
operatorImage:
11+
description: "Operator Image"
12+
default: "litmuschaos/chaos-operator:ci"
13+
required: true
14+
runnerImage:
15+
description: "Runner Image"
16+
default: "litmuschaos/chaos-runner:ci"
17+
required: true
18+
chaosNamespace:
19+
description: "Chaos Namespace"
20+
default: "default"
21+
required: true
22+
experimentImagePullPolicy:
23+
description: "Experiment Image Pull Policy"
24+
default: "Always"
25+
required: true
26+
27+
jobs:
28+
Azure_VM_Instance_Stop:
29+
runs-on: ubuntu-latest
30+
steps:
31+
32+
#Install and configure a k3s cluster
33+
- name: Installing Prerequisites (K3S Cluster)
34+
env:
35+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
36+
run: |
37+
curl -sfL https://get.k3s.io | sh -s - --docker --write-kubeconfig-mode 664
38+
kubectl wait node --all --for condition=ready --timeout=90s
39+
mkdir -p $HOME/.kube && cat /etc/rancher/k3s/k3s.yaml > $HOME/.kube/config
40+
kubectl get nodes
41+
42+
- uses: actions/checkout@v2
43+
44+
- uses: actions/setup-go@v2
45+
with:
46+
go-version: '1.16'
47+
48+
- name: Create azure auth file from secrets
49+
run: echo "${{ secrets.AZURE_AUTH_FILE }}" > azure.auth
50+
51+
- name: Create Kubernetes secret for azure experiment
52+
if: always()
53+
env:
54+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
55+
run: |
56+
kubectl create secret generic cloud-secret --from-file=azure.auth
57+
58+
- name: Azure Login
59+
uses: azure/login@v1
60+
with:
61+
creds: ${{ secrets.AZURE_CREDENTIALS }}
62+
63+
- name: Create target Azure VM Instances
64+
if: always()
65+
run: |
66+
az group create -n ${{ secrets.AZURE_RESOURCE_GROUP }}_instance_${{ github.run_number }} -l eastus && \
67+
az vm create -g ${{ secrets.AZURE_RESOURCE_GROUP }}_instance_${{ github.run_number }} -n litmus-e2e-instance-one --image UbuntuLTS --generate-ssh-keys && \
68+
az vm create -g ${{ secrets.AZURE_RESOURCE_GROUP }}_instance_${{ github.run_number }} -n litmus-e2e-instance-two --image UbuntuLTS --generate-ssh-keys
69+
70+
- name: Litmus Infra Setup
71+
if: always()
72+
run: make build-litmus
73+
env:
74+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
75+
OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}"
76+
RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}"
77+
78+
- name: Run Azure VM Instance Stop experiment in serial & parallel mode
79+
if: always()
80+
env:
81+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
82+
AZURE_INSTANCE_NAME: "litmus-e2e-instance-one,litmus-e2e-instance-two"
83+
AZURE_RESOURCE_GROUP : "${{ secrets.AZURE_RESOURCE_GROUP }}_instance_${{ github.run_number }}"
84+
GO_EXPERIMENT_IMAGE: "${{ github.event.inputs.goExperimentImage }}"
85+
EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}"
86+
CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}"
87+
run: make azure-instance-stop
88+
89+
- name: Delete target Azure VM Instances
90+
if: always()
91+
run: |
92+
az vm delete --ids $(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }}_instance_${{ github.run_number }} --query "[].id" -o tsv) --yes
93+
az group delete -n ${{ secrets.AZURE_RESOURCE_GROUP }}_instance_${{ github.run_number }} --yes
94+
95+
- name: "[Debug]: check chaos resources"
96+
if: ${{ failure() }}
97+
continue-on-error: true
98+
run: |
99+
bash <(curl -s https://raw.githubusercontent.com/litmuschaos/litmus-e2e/master/build/debug.sh)
100+
101+
- name: "[Debug]: check operator logs"
102+
if: ${{ failure() }}
103+
continue-on-error: true
104+
run: |
105+
operator_name=$(kubectl get pods -n litmus -l app.kubernetes.io/component=operator --no-headers | awk '{print$1}')
106+
kubectl logs $operator_name -n litmus > logs.txt
107+
cat logs.txt
108+
109+
- name: Deleting K3S cluster
110+
if: always()
111+
run: /usr/local/bin/k3s-uninstall.sh
112+
113+
Azure_VM_Disk_Loss:
114+
runs-on: ubuntu-latest
115+
steps:
116+
117+
#Install and configure a k3s cluster
118+
- name: Installing Prerequisites (K3S Cluster)
119+
env:
120+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
121+
run: |
122+
curl -sfL https://get.k3s.io | sh -s - --docker --write-kubeconfig-mode 664
123+
kubectl wait node --all --for condition=ready --timeout=90s
124+
mkdir -p $HOME/.kube && cat /etc/rancher/k3s/k3s.yaml > $HOME/.kube/config
125+
kubectl get nodes
126+
127+
- uses: actions/checkout@v2
128+
129+
- uses: actions/setup-go@v2
130+
with:
131+
go-version: '1.14'
132+
133+
- name: Create azure auth file from secrets
134+
run: echo "${{ secrets.AZURE_AUTH_FILE }}" > azure.auth
135+
136+
- name: Create Kubernetes secret for azure experiment
137+
if: always()
138+
env:
139+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
140+
run: |
141+
kubectl create secret generic cloud-secret --from-file=azure.auth
142+
143+
- name: Azure Login
144+
uses: azure/login@v1
145+
with:
146+
creds: ${{ secrets.AZURE_CREDENTIALS }}
147+
148+
- name: Create a Azure VM Instance with target Disk Volumes
149+
if: always()
150+
run: |
151+
az group create -n ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} -l eastus && \
152+
az vm create -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} -n litmus-e2e-instance-one --image UbuntuLTS --generate-ssh-keys && \
153+
az disk create -n litmus-e2e-disk-one -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --size-gb=4 && \
154+
az disk create -n litmus-e2e-disk-two -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --size-gb=4 && \
155+
az vm disk attach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-one && \
156+
az vm disk attach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-two
157+
- name: Litmus Infra Setup
158+
if: always()
159+
run: make build-litmus
160+
env:
161+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
162+
OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}"
163+
RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}"
164+
165+
- name: Run Azure VM Disk Loss experiment in serial & parallel mode
166+
if: always()
167+
env:
168+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
169+
AZURE_INSTANCE_NAME: "litmus-e2e-instance-one,litmus-e2e-instance-two"
170+
AZURE_DISK_NAME: "litmus-e2e-disk-one,litmus-e2e-disk-two"
171+
AZURE_RESOURCE_GROUP: "${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }}"
172+
GO_EXPERIMENT_IMAGE: "${{ github.event.inputs.goExperimentImage }}"
173+
EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}"
174+
CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}"
175+
run: make azure-disk-loss
176+
177+
- name: Delete the VM Instance and target Disk Volumes
178+
if: always()
179+
run: |
180+
az vm disk detach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-one && \
181+
az vm disk detach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-two && \
182+
az disk delete -n litmus-e2e-disk-one -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --yes && \
183+
az disk delete -n litmus-e2e-disk-two -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --yes && \
184+
az vm delete --ids $(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --query "[].id" -o tsv) --yes
185+
az group delete -n ${{ secrets.AZURE_RESOURCE_GROUP }}_disk_${{ github.run_number }} --yes
186+
187+
- name: "[Debug]: check chaos resources"
188+
if: ${{ failure() }}
189+
continue-on-error: true
190+
run: |
191+
bash <(curl -s https://raw.githubusercontent.com/litmuschaos/litmus-e2e/master/build/debug.sh)
192+
193+
- name: "[Debug]: check operator logs"
194+
if: ${{ failure() }}
195+
continue-on-error: true
196+
run: |
197+
operator_name=$(kubectl get pods -n litmus -l app.kubernetes.io/component=operator --no-headers | awk '{print$1}')
198+
kubectl logs $operator_name -n litmus > logs.txt
199+
cat logs.txt
200+
201+
- name: Deleting K3S cluster
202+
if: always()
203+
run: /usr/local/bin/k3s-uninstall.sh

‎.github/workflows/nightly-azure-experiment-pipeline.yml

+183
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
---
2+
name: Scheduled-Azure-Experiment-Pipeline
3+
on:
4+
schedule:
5+
- cron: "30 22 * * *" # Daily 02:30 AM in midnight
6+
7+
jobs:
8+
Azure_VM_Instance_Stop:
9+
runs-on: ubuntu-latest
10+
steps:
11+
12+
#Install and configure a k3s cluster
13+
- name: Installing Prerequisites (K3S Cluster)
14+
env:
15+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
16+
run: |
17+
curl -sfL https://get.k3s.io | sh -s - --docker --write-kubeconfig-mode 664
18+
kubectl wait node --all --for condition=ready --timeout=90s
19+
mkdir -p $HOME/.kube && cat /etc/rancher/k3s/k3s.yaml > $HOME/.kube/config
20+
kubectl get nodes
21+
22+
- uses: actions/checkout@v2
23+
24+
- uses: actions/setup-go@v2
25+
with:
26+
go-version: '1.16'
27+
28+
- name: Create azure auth file from secrets
29+
run: echo "${{ secrets.AZURE_AUTH_FILE }}" > azure.auth
30+
31+
- name: Create Kubernetes secret for azure experiment
32+
if: always()
33+
env:
34+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
35+
run: |
36+
kubectl create secret generic cloud-secret --from-file=azure.auth
37+
38+
- name: Azure Login
39+
uses: azure/login@v1
40+
with:
41+
creds: ${{ secrets.AZURE_CREDENTIALS }}
42+
43+
- name: Create target Azure VM Instances
44+
if: always()
45+
run: |
46+
az group create -n ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_instance_${{ github.run_number }} -l eastus && \
47+
az vm create -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_instance_${{ github.run_number }} -n litmus-e2e-instance-one --image UbuntuLTS --generate-ssh-keys && \
48+
az vm create -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_instance_${{ github.run_number }} -n litmus-e2e-instance-two --image UbuntuLTS --generate-ssh-keys
49+
50+
- name: Litmus Infra Setup
51+
if: always()
52+
run: make build-litmus
53+
env:
54+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
55+
OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}"
56+
RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}"
57+
58+
- name: Run Azure VM Instance Stop experiment in serial & parallel mode
59+
if: always()
60+
env:
61+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
62+
AZURE_INSTANCE_NAME: "litmus-e2e-instance-one,litmus-e2e-instance-two"
63+
AZURE_RESOURCE_GROUP : "${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_instance_${{ github.run_number }}"
64+
GO_EXPERIMENT_IMAGE: "${{ github.event.inputs.goExperimentImage }}"
65+
EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}"
66+
CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}"
67+
run: make azure-instance-stop
68+
69+
- name: Delete target Azure VM Instances
70+
if: always()
71+
run: |
72+
az vm delete --ids $(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_instance_${{ github.run_number }} --query "[].id" -o tsv) --yes
73+
az group delete -n ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_instance_${{ github.run_number }} --yes
74+
75+
- name: "[Debug]: check chaos resources"
76+
if: ${{ failure() }}
77+
continue-on-error: true
78+
run: |
79+
bash <(curl -s https://raw.githubusercontent.com/litmuschaos/litmus-e2e/master/build/debug.sh)
80+
81+
- name: "[Debug]: check operator logs"
82+
if: ${{ failure() }}
83+
continue-on-error: true
84+
run: |
85+
operator_name=$(kubectl get pods -n litmus -l app.kubernetes.io/component=operator --no-headers | awk '{print$1}')
86+
kubectl logs $operator_name -n litmus > logs.txt
87+
cat logs.txt
88+
89+
- name: Deleting K3S cluster
90+
if: always()
91+
run: /usr/local/bin/k3s-uninstall.sh
92+
93+
Azure_VM_Disk_Loss:
94+
runs-on: ubuntu-latest
95+
steps:
96+
97+
#Install and configure a k3s cluster
98+
- name: Installing Prerequisites (K3S Cluster)
99+
env:
100+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
101+
run: |
102+
curl -sfL https://get.k3s.io | sh -s - --docker --write-kubeconfig-mode 664
103+
kubectl wait node --all --for condition=ready --timeout=90s
104+
mkdir -p $HOME/.kube && cat /etc/rancher/k3s/k3s.yaml > $HOME/.kube/config
105+
kubectl get nodes
106+
107+
- uses: actions/checkout@v2
108+
109+
- uses: actions/setup-go@v2
110+
with:
111+
go-version: '1.14'
112+
113+
- name: Create azure auth file from secrets
114+
run: echo "${{ secrets.AZURE_AUTH_FILE }}" > azure.auth
115+
116+
- name: Create Kubernetes secret for azure experiment
117+
if: always()
118+
env:
119+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
120+
run: |
121+
kubectl create secret generic cloud-secret --from-file=azure.auth
122+
123+
- name: Azure Login
124+
uses: azure/login@v1
125+
with:
126+
creds: ${{ secrets.AZURE_CREDENTIALS }}
127+
128+
- name: Create a Azure VM Instance with target Disk Volumes
129+
if: always()
130+
run: |
131+
az group create -n ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} -l eastus && \
132+
az vm create -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} -n litmus-e2e-instance-one --image UbuntuLTS --generate-ssh-keys && \
133+
az disk create -n litmus-e2e-disk-one -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --size-gb=4 && \
134+
az disk create -n litmus-e2e-disk-two -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --size-gb=4 && \
135+
az vm disk attach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-one && \
136+
az vm disk attach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-two
137+
- name: Litmus Infra Setup
138+
if: always()
139+
run: make build-litmus
140+
env:
141+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
142+
OPERATOR_IMAGE: "${{ github.event.inputs.operatorImage }}"
143+
RUNNER_IMAGE: "${{ github.event.inputs.runnerImage }}"
144+
145+
- name: Run Azure VM Disk Loss experiment in serial & parallel mode
146+
if: always()
147+
env:
148+
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
149+
AZURE_INSTANCE_NAME: "litmus-e2e-instance-one,litmus-e2e-instance-two"
150+
AZURE_DISK_NAME: "litmus-e2e-disk-one,litmus-e2e-disk-two"
151+
AZURE_RESOURCE_GROUP: "${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }}"
152+
GO_EXPERIMENT_IMAGE: "${{ github.event.inputs.goExperimentImage }}"
153+
EXPERIMENT_IMAGE_PULL_POLICY: "${{ github.event.inputs.experimentImagePullPolicy }}"
154+
CHAOS_NAMESPACE: "${{ github.event.inputs.chaosNamespace }}"
155+
run: make azure-disk-loss
156+
157+
- name: Delete the VM Instance and target Disk Volumes
158+
if: always()
159+
run: |
160+
az vm disk detach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-one && \
161+
az vm disk detach -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --vm-name litmus-e2e-instance-one -n litmus-e2e-disk-two && \
162+
az disk delete -n litmus-e2e-disk-one -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --yes && \
163+
az disk delete -n litmus-e2e-disk-two -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --yes && \
164+
az vm delete --ids $(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --query "[].id" -o tsv) --yes
165+
az group delete -n ${{ secrets.AZURE_RESOURCE_GROUP }}_schedule_disk_${{ github.run_number }} --yes
166+
167+
- name: "[Debug]: check chaos resources"
168+
if: ${{ failure() }}
169+
continue-on-error: true
170+
run: |
171+
bash <(curl -s https://raw.githubusercontent.com/litmuschaos/litmus-e2e/master/build/debug.sh)
172+
173+
- name: "[Debug]: check operator logs"
174+
if: ${{ failure() }}
175+
continue-on-error: true
176+
run: |
177+
operator_name=$(kubectl get pods -n litmus -l app.kubernetes.io/component=operator --no-headers | awk '{print$1}')
178+
kubectl logs $operator_name -n litmus > logs.txt
179+
cat logs.txt
180+
181+
- name: Deleting K3S cluster
182+
if: always()
183+
run: /usr/local/bin/k3s-uninstall.sh

0 commit comments

Comments
 (0)