Skip to content

Commit a3a0dc0

Browse files
elronbandelelronbandel
committed
Move system format tests to dedicated file
Signed-off-by: Elron Bandel <[email protected]>
1 parent 901c6c8 commit a3a0dc0

File tree

1 file changed

+0
-241
lines changed

1 file changed

+0
-241
lines changed

‎tests/test_operators.py‎

Lines changed: 0 additions & 241 deletions
Original file line numberDiff line numberDiff line change
@@ -263,247 +263,6 @@ def test_flatten_instances(self):
263263
tester=self,
264264
)
265265

266-
def test_system_format(self):
267-
demo_instances = [
268-
{"source": "1+2", "target": "3"},
269-
{"source": "4-2", "target": "2"},
270-
]
271-
instruction = "solve the math exercises"
272-
273-
inputs = [
274-
{
275-
"source": "1+1",
276-
"source1": "1+1",
277-
"target": "2",
278-
"instruction": instruction,
279-
"demos": demo_instances,
280-
},
281-
{
282-
"source": "3+2",
283-
"source1": "3+2",
284-
"target": "5",
285-
"instruction": instruction,
286-
"demos": demo_instances,
287-
},
288-
{
289-
"source": "7-4",
290-
"source1": "7-4",
291-
"target": "3",
292-
"instruction": instruction,
293-
"demos": demo_instances,
294-
},
295-
{
296-
"source": "12-3",
297-
"source1": "12-3",
298-
"target": "9",
299-
"instruction": instruction,
300-
"demos": demo_instances,
301-
},
302-
]
303-
304-
# imitating iclformat's add_instruction_after_demos=True, instruction is not "", and target_prefix =""
305-
system_format = SystemFormat(
306-
demos_field="demos",
307-
demo_format="User: {source}\nAgent: {target}\n\n",
308-
model_input_format="{demos}User: {instruction}\n\n{source}\nAgent: ",
309-
)
310-
311-
targets = [
312-
{
313-
"source1": "1+1",
314-
"target": "2",
315-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: solve the math exercises\n\n1+1\nAgent: ",
316-
},
317-
{
318-
"source1": "3+2",
319-
"target": "5",
320-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: solve the math exercises\n\n3+2\nAgent: ",
321-
},
322-
{
323-
"source1": "7-4",
324-
"target": "3",
325-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: solve the math exercises\n\n7-4\nAgent: ",
326-
},
327-
{
328-
"source1": "12-3",
329-
"target": "9",
330-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: solve the math exercises\n\n12-3\nAgent: ",
331-
},
332-
]
333-
334-
check_operator(
335-
operator=system_format,
336-
inputs=inputs,
337-
targets=targets,
338-
tester=self,
339-
)
340-
341-
# now imitate instruction before demos.
342-
system_format = SystemFormat(
343-
demos_field="demos",
344-
demo_format="User: {source}\nAgent: {target}\n\n",
345-
model_input_format="Instruction: {instruction}\n\n{demos}User: {source}\nAgent: ",
346-
)
347-
348-
targets = [
349-
{
350-
"source1": "1+1",
351-
"target": "2",
352-
"source": "Instruction: solve the math exercises\n\nUser: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 1+1\nAgent: ",
353-
},
354-
{
355-
"source1": "3+2",
356-
"target": "5",
357-
"source": "Instruction: solve the math exercises\n\nUser: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 3+2\nAgent: ",
358-
},
359-
{
360-
"source1": "7-4",
361-
"target": "3",
362-
"source": "Instruction: solve the math exercises\n\nUser: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 7-4\nAgent: ",
363-
},
364-
{
365-
"source1": "12-3",
366-
"target": "9",
367-
"source": "Instruction: solve the math exercises\n\nUser: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 12-3\nAgent: ",
368-
},
369-
]
370-
371-
check_operator(
372-
operator=system_format,
373-
inputs=inputs,
374-
targets=targets,
375-
tester=self,
376-
)
377-
378-
# test with instruction = "":
379-
for instance in inputs:
380-
instance.pop("instruction")
381-
382-
system_format = SystemFormat(
383-
demos_field="demos",
384-
demo_format="User: {source}\nAgent: {target}\n\n",
385-
model_input_format="{demos}User: {instruction}{source}\nAgent: ",
386-
)
387-
388-
targets_no_instruction = [
389-
{
390-
"source1": "1+1",
391-
"target": "2",
392-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 1+1\nAgent: ",
393-
},
394-
{
395-
"source1": "3+2",
396-
"target": "5",
397-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 3+2\nAgent: ",
398-
},
399-
{
400-
"source1": "7-4",
401-
"target": "3",
402-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 7-4\nAgent: ",
403-
},
404-
{
405-
"source1": "12-3",
406-
"target": "9",
407-
"source": "User: 1+2\nAgent: 3\n\nUser: 4-2\nAgent: 2\n\nUser: 12-3\nAgent: ",
408-
},
409-
]
410-
411-
check_operator(
412-
operator=system_format,
413-
inputs=inputs,
414-
targets=targets_no_instruction,
415-
tester=self,
416-
)
417-
418-
# ICLFormat tests from tests_renderers, migrated here
419-
instance = {
420-
"source": 'This is my sentence: "was so bad"',
421-
"target": "negative",
422-
"references": ["negative"],
423-
"instruction": "classify user sentence by its sentiment to either positive, or negative.",
424-
"demos": [
425-
{
426-
"source": 'This is my sentence: "was so not good"',
427-
"target": "negative",
428-
"references": ["negative"],
429-
},
430-
{
431-
"source": 'This is my sentence: "was so good"',
432-
"target": "positive",
433-
"references": ["positive"],
434-
},
435-
],
436-
}
437-
438-
system_format = SystemFormat(
439-
demo_format="User:{source}\nAgent:{target}\n\n",
440-
model_input_format="Instruction:{instruction}\n\n{demos}User:{source}\nAgent:",
441-
)
442-
443-
result = system_format.process(instance)
444-
445-
target = {
446-
"source": 'Instruction:classify user sentence by its sentiment to either positive, or negative.\n\nUser:This is my sentence: "was so not good"\nAgent:negative\n\nUser:This is my sentence: "was so good"\nAgent:positive\n\nUser:This is my sentence: "was so bad"\nAgent:',
447-
"target": "negative",
448-
"references": ["negative"],
449-
}
450-
self.assertDictEqual(result, target)
451-
452-
# no demos
453-
instance = {
454-
"source": 'This is my sentence: "was so bad"',
455-
"target": "negative",
456-
"references": ["negative"],
457-
"instruction": "classify user sentence by its sentiment to either positive, or negative.",
458-
}
459-
system_format = SystemFormat(
460-
demo_format="User:{source}\nAgent:{target}\n\n",
461-
model_input_format="Instruction:{instruction}\n\n{demos}User:{source}\nAgent:",
462-
)
463-
result = system_format.process(instance)
464-
target = {
465-
"source": 'Instruction:classify user sentence by its sentiment to either positive, or negative.\n\nUser:This is my sentence: "was so bad"\nAgent:',
466-
"target": "negative",
467-
"references": ["negative"],
468-
}
469-
self.assertDictEqual(result, target)
470-
471-
# test_system_format_with_prefix_and_suffix(self):
472-
system_format_fix = SystemFormat(
473-
demos_field="demos",
474-
demo_format="User: {source}\nAgent: {target}\n\n",
475-
model_input_format="[INST] <<SYS>>\n{instruction}\n\n{demos}User: {source}\nAgent: [/INST]",
476-
)
477-
renderer = system_format_fix
478-
479-
instance = {
480-
"source": 'This is my sentence: "was so bad"',
481-
"target": "negative",
482-
"references": ["negative"],
483-
"instruction": "classify user sentence by its sentiment to either positive, or negative.",
484-
"demos": [
485-
{
486-
"source": 'This is my sentence: "was so not good"',
487-
"target": "negative",
488-
"references": ["negative"],
489-
},
490-
{
491-
"source": 'This is my sentence: "was so good"',
492-
"target": "positive",
493-
"references": ["positive"],
494-
},
495-
],
496-
}
497-
self.maxDiff = None
498-
result = renderer.process(instance)
499-
target = {
500-
"source": '[INST] <<SYS>>\nclassify user sentence by its sentiment to either positive, or negative.\n\nUser: This is my sentence: "was so not good"\nAgent: negative\n\nUser: This is my sentence: "was so good"\nAgent: positive\n\nUser: This is my sentence: "was so bad"\nAgent: [/INST]',
501-
"target": "negative",
502-
"references": ["negative"],
503-
}
504-
505-
self.assertDictEqual(result, target)
506-
507266
def test_filter_by_values_with_required_values(self):
508267
inputs = [{"a": 1, "b": 2}, {"a": 2, "b": 3}, {"a": 1, "b": 3}]
509268

0 commit comments

Comments
 (0)