Skip to content

Commit f5e3759

Browse files
ijusttookadnatestijusttookadnatest
committed
feat: improve error displaying when snak test failures
1 parent 9632f5d commit f5e3759

File tree

5 files changed

+140
-103
lines changed

5 files changed

+140
-103
lines changed

‎.github/workflows/backend.yml‎

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -123,15 +123,15 @@ jobs:
123123
echo "Waiting for services to be ready..."
124124
sleep 20
125125
126-
chmod +x ./tests/integration-tests.sh
127-
chmod +x ./tests/database-connection.sh
126+
chmod +x ./scripts/integration-tests.sh
127+
chmod +x ./scripts/database-connection.sh
128128
129129
echo -e "\n=== Running basic integration tests ==="
130-
./tests/integration-tests.sh
130+
./scripts/integration-tests.sh
131131
INTEGRATION_RESULT=$?
132132
133133
echo -e "\n=== Running database connection test via chat/completions endpoint ==="
134-
./tests/database-connection.sh
134+
./scripts/database-connection.sh
135135
DB_CONNECTION_RESULT=$?
136136
137137
if [ $INTEGRATION_RESULT -ne 0 ] || [ $DB_CONNECTION_RESULT -ne 0 ]; then
@@ -215,18 +215,8 @@ jobs:
215215
216216
- name: Run cairo code generation test
217217
run: |
218-
pnpm run test:code-quality
219-
220-
TEST_RESULT=$?
221-
222-
pkill -f "start:server" || true
223-
224-
if [ $TEST_RESULT -ne 0 ]; then
225-
echo "❌ API test failed!"
226-
exit 1
227-
else
228-
echo "✅ API test passed!"
229-
fi
218+
chmod +x ./scripts/snak-test.sh
219+
bash scripts/snak-test.sh
230220
231221
# - name: Push docker image
232222
# run: docker push ${{ github.repository }}:${{ github.sha }}

‎packages/agents/__tests__/code-quality/snak.test.ts‎

Lines changed: 120 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -18,97 +18,130 @@ describe('Code Generation and Compilation Tests', () => {
1818
project_name: string,
1919
prompt_content: string,
2020
index: number,
21-
): Promise<boolean> {
22-
const generation_prompt = `Test #${index}: Generate Cairo code for ${prompt_content}
23-
24-
1. First, register a new project named "${project_name}" using the cairocoder_register_project tool
25-
2. Then, generate the Cairo code using the cairocoder_generate_code tool
26-
27-
If generation is successful:
28-
- Return the generated Cairo code with syntax highlighting
29-
30-
If generation fails:
31-
- Return only the error message from the tool
32-
- Do not try to fix or retry the generation
33-
34-
Do not perform any additional actions.`;
35-
const generateResponse = await agent
36-
.post('/api/key/request')
37-
.set('Content-Type', 'application/json')
38-
.set('x-api-key', API_KEY)
39-
.send({
40-
request: generation_prompt,
41-
});
42-
43-
console.log(
44-
'GENERATION RESPONSE:',
45-
JSON.stringify(generateResponse.body, null, 2),
46-
);
47-
const sucessfulGeneration = generateResponse.body.output[0].text
48-
.toLowerCase()
49-
.includes('```cairo');
50-
51-
if (
52-
generateResponse.body.output[0].status !== 'success' ||
53-
!sucessfulGeneration
54-
) {
55-
console.error('Generation failed:', generateResponse.body);
56-
return false;
57-
}
58-
59-
console.log('Generated code successfully');
60-
61-
const compilation_prompt = `Test #${index}: Compile the project "${project_name}" using the scarb_compile_contract tool.
62-
63-
After compilation, report whether it succeeded or failed.
64-
65-
For successful compilation: Report "Compilation successful" and include any relevant output.
66-
For failed compilation: Report "Compilation failed" and include the specific error messages.
67-
68-
Only use the compilation tool and no other tools.
69-
If another tool is used, instead or additionally to the compilation tool, report it as a failure.`;
70-
71-
const compileResponse = await agent
72-
.post('/api/key/request')
73-
.set('Content-Type', 'application/json')
74-
.set('x-api-key', API_KEY)
75-
.send({
76-
request: compilation_prompt,
77-
});
78-
79-
console.log(
80-
'COMPILATION RESPONSE:',
81-
JSON.stringify(compileResponse.body, null, 2),
82-
);
83-
84-
const sucessfulCompilation =
85-
compileResponse.body.output[0].text
21+
): Promise<{success: boolean; error?: string}> {
22+
console.log(`\n=== Test #${index}: ${project_name} ===`);
23+
console.log(`Generating code for: ${prompt_content}`);
24+
25+
try {
26+
const generation_prompt = `Test #${index}: Generate Cairo code for ${prompt_content}
27+
28+
1. First, register a new project named "${project_name}" using the cairocoder_register_project tool
29+
2. Then, generate the Cairo code using the cairocoder_generate_code tool
30+
31+
If generation is successful:
32+
- Return the generated Cairo code with syntax highlighting
33+
34+
If generation fails:
35+
- Return only the error message from the tool
36+
- Do not try to fix or retry the generation
37+
38+
Do not perform any additional actions.`;
39+
const generateResponse = await agent
40+
.post('/api/key/request')
41+
.set('Content-Type', 'application/json')
42+
.set('x-api-key', API_KEY)
43+
.send({
44+
request: generation_prompt,
45+
});
46+
47+
console.log('CODE GENERATION STATUS:', generateResponse.status);
48+
49+
if (generateResponse.status !== 201) {
50+
return {
51+
success: false,
52+
error: `Generation HTTP request failed with status ${generateResponse.status}: ${JSON.stringify(generateResponse.body)}`
53+
};
54+
}
55+
56+
console.log('CODE GENERATION RESPONSE:', JSON.stringify(generateResponse.body.output[0], null, 2));
57+
const sucessfulGeneration = generateResponse.body.output[0].text
8658
.toLowerCase()
87-
.includes('compilation') &&
88-
!compileResponse.body.output[0].text.toLowerCase().includes('failure') &&
89-
!compileResponse.body.output[0].text.toLowerCase().includes('failed') &&
90-
!compileResponse.body.output[0].text.toLowerCase().includes('error');
91-
if (
92-
compileResponse.body.output[0].status !== 'success' ||
93-
!sucessfulCompilation
94-
) {
95-
console.error('Compilation request failed:', compileResponse.body);
96-
return false;
97-
}
98-
99-
console.log('END REQUEST ////////');
100-
await new Promise((resolve) => setTimeout(resolve, 5000));
101-
102-
return true;
59+
.includes('```cairo');
60+
61+
if (
62+
generateResponse.body.output[0].status !== 'success' ||
63+
!sucessfulGeneration
64+
) {
65+
return {
66+
success: false,
67+
error: `Generation failed: ${JSON.stringify(generateResponse.body.output[0].text)}`
68+
};
69+
}
70+
71+
console.log('✅ Code generated successfully');
72+
73+
const compilation_prompt = `Test #${index}: Compile the project "${project_name}" using the scarb_compile_contract tool.
74+
75+
After compilation, report whether it succeeded or failed.
76+
77+
For successful compilation: Report "Compilation successful" and include any relevant output.
78+
For failed compilation: Report "Compilation failed" and include the specific error messages.
79+
80+
Only use the compilation tool and no other tools.
81+
If another tool is used, instead or additionally to the compilation tool, report it as a failure.`;
82+
83+
const compileResponse = await agent
84+
.post('/api/key/request')
85+
.set('Content-Type', 'application/json')
86+
.set('x-api-key', API_KEY)
87+
.send({
88+
request: compilation_prompt,
89+
});
90+
91+
92+
console.log('COMPILATION STATUS:', compileResponse.status);
93+
94+
if (compileResponse.status !== 201) {
95+
return {
96+
success: false,
97+
error: `Compilation HTTP request failed with status ${compileResponse.status}: ${JSON.stringify(compileResponse.body)}`
98+
};
99+
}
100+
101+
console.log('COMPILATION RESPONSE:', JSON.stringify(compileResponse.body.output[0], null, 2));
102+
103+
const sucessfulCompilation =
104+
compileResponse.body.output[0].text
105+
.toLowerCase()
106+
.includes('compilation') &&
107+
!compileResponse.body.output[0].text.toLowerCase().includes('failure') &&
108+
!compileResponse.body.output[0].text.toLowerCase().includes('failed') &&
109+
!compileResponse.body.output[0].text.toLowerCase().includes('error');
110+
111+
if (
112+
compileResponse.body.output[0].status !== 'success' ||
113+
!sucessfulCompilation
114+
) {
115+
return {
116+
success: false,
117+
error: `Compilation failed: ${JSON.stringify(compileResponse.body.output[0].text)}`
118+
};
119+
}
120+
121+
console.log('✅ Compilation successful');
122+
await new Promise((resolve) => setTimeout(resolve, 5000));
123+
124+
return { success: true };
125+
} catch (error) {
126+
console.error(`❌ Unexpected error in Test #${index}:`, error);
127+
return {
128+
success: false,
129+
error: `Unexpected error: ${error.message}`
130+
};
103131
}
132+
}
104133

105134
describe('Cairo Functions and Basic Algorithms', () => {
106-
test('Factorial function', async () => {
107-
// const project_name = 'factorial';
108-
// const prompt_content =
109-
// 'a Cairo function that calculates the factorial of a number';
110-
// const success = await generateAndCompile(project_name, prompt_content, 1);
111-
// expect(success).toBe(true);
135+
test('Fibonacci function', async () => {
136+
const project_name = 'fibonacci';
137+
const prompt_content = 'a Cairo function that calculates the Fibonacci sequence';
138+
const result = await generateAndCompile(project_name, prompt_content, 1);
139+
140+
if (!result.success) {
141+
console.error(`❌ TEST FAILED: ${result.error}`);
142+
}
143+
144+
expect(result.success).toBe(true);
112145
}, 100000);
113146

114147
// test('Max value in array', async () => {

‎tests/database-connection.sh‎ renamed to ‎scripts/database-connection.sh‎

File renamed without changes.

‎tests/integration-tests.sh‎ renamed to ‎scripts/integration-tests.sh‎

File renamed without changes.

‎scripts/snak-test.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
pnpm run test:code-quality --force | tee test-output.log
4+
5+
TEST_RESULT=${PIPESTATUS[0]}
6+
7+
if [ $TEST_RESULT -ne 0 ]; then
8+
echo "❌ API test failed with exit code $TEST_RESULT!"
9+
echo "=============== Test Logs ==============="
10+
grep -A 5 -E "TEST FAILED|❌" test-output.log || true
11+
exit 1
12+
else
13+
echo "✅ API test passed!"
14+
fi

0 commit comments

Comments
 (0)