-
Notifications
You must be signed in to change notification settings - Fork 13
165 lines (150 loc) · 5.11 KB
/
ci.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
name: Test
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
milvus-cdc-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: 3.8
cache: pip
cache-dependency-path: 'tests/requirements.txt'
- name: Deploy Source Milvus
timeout-minutes: 15
shell: bash
working-directory: tests/deployment/upstream/
run: |
docker-compose up -d
bash ../../../scripts/check_healthy.sh
docker-compose ps -a
- name: Creating kind cluster
uses: helm/[email protected]
- name: Print cluster information
run: |
kubectl config view
kubectl cluster-info
kubectl get nodes
kubectl get pods -n kube-system
helm version
kubectl version
- name: Deploy Target Milvus
timeout-minutes: 15
shell: bash
working-directory: tests/deployment/downstream
run: |
helm repo add milvus https://milvus-io.github.io/milvus-helm
helm repo update
helm install --wait --timeout 720s cdc-target milvus/milvus -f standalone-values.yaml;
kubectl get pods
sleep 20s
kubectl get pods
kubectl port-forward service/cdc-target-milvus 19500:19530 >/dev/null 2>&1 &
sleep 20s
# check whether port-forward success
nc -vz 127.0.0.1 19500
- uses: actions/setup-go@v4
with:
go-version: '1.18'
cache-dependency-path: server/go.sum
cache: true
- name: Build CDC
timeout-minutes: 15
working-directory: server
shell: bash
run: |
go build -o milvus-cdc main/main.go
ls -l
- name: Deploy Milvus CDC
timeout-minutes: 15
working-directory: server
shell: bash
run: |
./milvus-cdc > server.log 2>&1 &
sleep 20s
nc -vz 127.0.0.1 8444
sleep 20s
curl --location '127.0.0.1:8444/cdc' \
--header 'Content-Type: application/json' \
--data '{
"request_type": "create",
"request_data": {
"milvus_connect_param": {
"host": "127.0.0.1",
"port": 19500,
"username": "",
"password": "",
"enable_tls": false,
"ignore_partition": true,
"connect_timeout": 10
},
"collection_infos": [
{
"name": "sift_128_euclidean_HNSW"
}
]
}
}'
- name: Install dependencies
timeout-minutes: 5
working-directory: tests
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Download data
timeout-minutes: 5
working-directory: tests/assets/ann_hdf5
run: |
bash download.sh
- name: Run tests for source milvus
timeout-minutes: 15
working-directory: tests/scripts
run: |
python source_test.py --host 127.0.0.1 --port 19530
- name: Wait for 1 minutes
timeout-minutes: 5
run: |
sleep 60s
- name: Run tests for target milvus
timeout-minutes: 15
working-directory: tests/scripts
run: |
python target_test.py --host 127.0.0.1 --port 19500
- name: Verify deletion
timeout-minutes: 15
working-directory: tests/scripts
run: |
python verify_delete.py --source_host 127.0.0.1 --source_port 19530 --target_host 127.0.0.1 --target_port 19500
- name: Run testcases
timeout-minutes: 30
working-directory: tests/testcases
run: |
pytest -s -v --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500
- name: Export upstream milvus logs
if: ${{ ! success() }}
timeout-minutes: 5
working-directory: tests/deployment/upstream
run: |
docker-compose ps -a
bash ../../../scripts/export_log_docker.sh
- name: Export downstream milvus logs
if: ${{ ! success() }}
timeout-minutes: 5
working-directory: scripts
run: |
bash export_log_k8s.sh default cdc-target k8s_logs
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v2
with:
name: logs
path: |
scripts/k8s_logs
tests/deployment/upstream/logs
server/server.log