Skip to content

Commit d3a9d56

Browse files
AlyaGomaaAlyaGomaa
committed
Merge remote-tracking branch 'origin/main'
2 parents 0e747b5 + c62cf57 commit d3a9d56

37 files changed

+7650
-18
lines changed

‎llm-unittest/.attic/test_grade.yaml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
description: Test if Ollama generates valid JSON actions with correct structure
2+
3+
providers:
4+
- id: ollama:smollm2:1.7b
5+
config:
6+
num_predict: 1024
7+
- id: ollama:gemma3:1b
8+
config:
9+
num_predict: 1024
10+
- id: ollama:llama3.2:1b
11+
config:
12+
num_predict: 1024
13+
- id: ollama:llama3.2:3b
14+
config:
15+
num_predict: 1024
16+
- id: ollama:phi4-mini:latest
17+
config:
18+
num_predict: 1024
19+
- id: ollama:qwen2.5:1.5b
20+
config:
21+
num_predict: 1024
22+
- id: ollama:qwen2.5:3b
23+
config:
24+
num_predict: 1024
25+
26+
prompts:
27+
- 'What is {{topic}}?'
28+
tests:
29+
- vars:
30+
topic: quantum computing
31+
assert:
32+
- type: model-graded-closedqa
33+
value: Explains the concept without using technical jargon
34+
- type: model-graded-closedqa
35+
value: Includes a practical real-world example
36+
37+
defaultTest:
38+
options:
39+
provider: ollama:qwen2.5-coder:latest

‎llm-unittest/.attic/test_json.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
description: Test if Ollama generates valid JSON
2+
3+
providers:
4+
- id: ollama:smollm2:1.7b
5+
config:
6+
num_predict: 1024
7+
8+
9+
prompts:
10+
- "Output a JSON object that contains the keys `color` and `countries`, describing the following object: {{item}}"
11+
12+
tests:
13+
- vars:
14+
item: Banana
15+
assert:
16+
- type: is-json
17+
value:
18+
required: ["color", "countries"]
19+
type: object
20+
properties:
21+
color:
22+
type: string
23+
countries:
24+
type: array
25+
items:
26+
type: string

‎llm-unittest/.attic/test_tool_use_ollama.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
2+
prompts:
3+
- 'What is the weather like in {{city}}?'
4+
5+
providers:
6+
- id: 'ollama:llama3.2:3b'
7+
label: 'functions-defined-in-config'
8+
config:
9+
function:
10+
[
11+
{
12+
'name': 'get_current_weather',
13+
'description': 'Get the current weather in a given location',
14+
'parameters':
15+
{
16+
'type': 'object',
17+
'properties':
18+
{
19+
'location':
20+
{
21+
'type': 'string',
22+
'description': 'The city and state, e.g. San Francisco, CA',
23+
},
24+
'unit': { 'type': 'string', 'enum': ['celsius', 'fahrenheit'] },
25+
},
26+
'required': ['location'],
27+
},
28+
},
29+
]
30+
31+
tests:
32+
- vars:
33+
city: Boston
34+
assert:
35+
- type: is-json
36+
- type: javascript
37+
value: output.name === 'get_current_weather'

‎llm-unittest/01_test_action_json_parsing.yaml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
description: Field Extraction from Networking JSON
2+
3+
providers:
4+
- id: ollama:smollm2:1.7b
5+
config:
6+
num_predict: 2048
7+
- id: ollama:gemma3:1b
8+
config:
9+
num_predict: 2048
10+
- id: ollama:llama3.2:1b
11+
config:
12+
num_predict: 2048
13+
- id: ollama:llama3.2:3b
14+
config:
15+
num_predict: 2048
16+
- id: ollama:phi4-mini:latest
17+
config:
18+
num_predict: 2048
19+
- id: ollama:qwen2.5:1.5b
20+
config:
21+
num_predict: 2048
22+
- id: ollama:qwen2.5:3b
23+
config:
24+
num_predict: 2048
25+
- id: ollama:granite3.1-dense:2b
26+
config:
27+
num_predict: 2048
28+
29+
30+
prompts:
31+
- |
32+
Given the following JSON:
33+
{{json}}
34+
35+
{{question}}
36+
37+
tests:
38+
- vars:
39+
json: '{"action":"ScanNetwork", "parameters": {"target_network":"1.1.1.0/24", "source_host": "2.2.2.2"}}'
40+
question: "What is the IP address of the network you want to scan?"
41+
assert:
42+
- type: contains
43+
value: "1.1.1.0/24"
44+
45+
- vars:
46+
json: '{"action":"ScanServices", "parameters": {"target_host":"2.2.2.3", "source_host": "2.2.2.2"}}'
47+
question: "Which host are we scanning services on?"
48+
assert:
49+
- type: contains
50+
value: "2.2.2.3"
51+
52+
- vars:
53+
json: '{"action":"ExploitService", "parameters":{"target_host":"1.1.1.1", "target_service":"openssh", "source_host": "1.1.1.2"}}'
54+
question: "What service is being exploited?"
55+
assert:
56+
- type: contains
57+
value: "openssh"
58+
59+
- vars:
60+
json: '{"action":"FindData", "parameters":{"target_host":"1.1.1.1", "source_host": "1.1.1.2"}}'
61+
question: "From which machine is the data search being initiated?"
62+
assert:
63+
- type: contains
64+
value: "1.1.1.2"
65+
66+
- vars:
67+
json: '{"action":"ExfiltrateData", "parameters": {"target_host": "2.2.2.2", "data": {"owner":"User1", "id":"WebData"}, "source_host": "1.1.1.2"}}'
68+
question: "Who owns the data being exfiltrated?"
69+
assert:
70+
- type: contains
71+
value: "User1"
72+

‎llm-unittest/02_test_action_json_understanding.yaml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
description: Summarize Networking Actions from JSON
2+
3+
providers:
4+
- id: ollama:smollm2:1.7b
5+
config:
6+
num_predict: 2048
7+
- id: ollama:gemma3:1b
8+
config:
9+
num_predict: 2048
10+
- id: ollama:llama3.2:1b
11+
config:
12+
num_predict: 2048
13+
- id: ollama:llama3.2:3b
14+
config:
15+
num_predict: 2048
16+
- id: ollama:phi4-mini:latest
17+
config:
18+
num_predict: 2048
19+
- id: ollama:qwen2.5:1.5b
20+
config:
21+
num_predict: 2048
22+
- id: ollama:qwen2.5:3b
23+
config:
24+
num_predict: 2048
25+
- id: ollama:granite3.1-dense:2b
26+
config:
27+
num_predict: 2048
28+
29+
prompts:
30+
- |
31+
Given the following JSON, describe the networking action in simple English:
32+
{{json}}
33+
34+
example
35+
JSON action {"action":"ScanNetwork", "parameters": {"target_network":"1.1.1.0/24", "source_host": "2.2.2.2"}}
36+
Answer "Scan a Network targeting 1.1.1.0/24 from 2.2.2.2"
37+
tests:
38+
- vars:
39+
json: '{"action":"ScanNetwork", "parameters": {"target_network":"1.1.1.0/24", "source_host": "2.2.2.2"}}'
40+
assert:
41+
- type: llm-rubric
42+
value: "Does the explanation describe a ScanNetwork action targeting the network 1.1.1.0/24, initiated from host 2.2.2.2?"
43+
44+
- vars:
45+
json: '{"action":"ScanServices", "parameters": {"target_host":"2.2.2.3", "source_host": "2.2.2.2"}}'
46+
assert:
47+
- type: llm-rubric
48+
value: "Does the explanation describe a Find Services action performed on host 2.2.2.3, initiated from 2.2.2.2?"
49+
50+
- vars:
51+
json: '{"action":"ExploitService", "parameters":{"target_host":"1.1.1.1", "target_service":"openssh", "source_host": "1.1.1.2"}}'
52+
assert:
53+
- type: llm-rubric
54+
value: "Does the explanation describe an ExploitService action targeting the 'openssh' service on host 1.1.1.1, initiated from 1.1.1.2?"
55+
56+
- vars:
57+
json: '{"action":"FindData", "parameters":{"target_host":"1.1.1.1", "source_host": "1.1.1.2"}}'
58+
assert:
59+
- type: llm-rubric
60+
value: "Does the explanation describe a FindData action performed on host 1.1.1.1 from source host 1.1.1.2?"
61+
62+
- vars:
63+
json: '{"action":"ExfiltrateData", "parameters": {"target_host": "2.2.2.2", "data": {"owner":"User1", "id":"WebData"}, "source_host": "1.1.1.2"}}'
64+
assert:
65+
- type: llm-rubric
66+
value: "Does the answer correctly explain that the action is to exfiltrate data from source host 1.1.1.2 to target host 2.2.2.2, involving data owned by User1 with ID 'WebData'?"
67+
68+
defaultTest:
69+
options:
70+
provider: ollama:qwen2.5-coder:latest
71+

0 commit comments

Comments
 (0)