Changing data folder structure

Author: apiad

score.py score_test.pyscore.py renamed to score_test.py

@@ -81,8 +81,9 @@ def update(dict_1, dict_2):
         submit = os.path.join(sys.argv[1], 'res')
         output = sys.argv[2]
     else:
-        gold = 'gold'
-        submit = 'submit'
+        print('(!) Using `trial/submit` as test files and `trial/gold` as reference files.')
+        gold = 'trial/gold'
+        submit = 'trial/submit'
         output = '.'
 
     totals1 = collections.defaultdict(lambda: 0)
@@ -102,9 +103,9 @@ def update(dict_1, dict_2):
             scenario3 = evaluate_3(name, gold, submit)
             update(scenario3, totals3)
 
-    pprint.pprint(('Scenario 1', totals1))
-    pprint.pprint(('Scenario 2', totals2))
-    pprint.pprint(('Scenario 3', totals3))
+    # pprint.pprint(('Scenario 1', totals1))
+    # pprint.pprint(('Scenario 2', totals2))
+    # pprint.pprint(('Scenario 3', totals3))
 
     correct_1 = sum([totals1['correct_A'], totals1['correct_B'], totals1['correct_C'], 0.5 * totals1['partial_A']])
     subtotal_1 = sum([totals1['partial_A'], totals1['correct_A'], totals1['correct_B'], totals1['incorrect_B'], totals1['correct_C']])
@@ -129,6 +130,17 @@ def update(dict_1, dict_2):
 
     macro = sum([abc_f1, bc_f1, c_f1]) / 3
 
+    print('abc_prec:%.5f' % abc_prec)
+    print('abc_rec:%.5f' % abc_rec)
+    print('abc_f1:%.5f' % abc_f1)
+    print('bc_prec:%.5f' % bc_prec)
+    print('bc_rec:%.5f' % bc_rec)
+    print('bc_f1:%.5f' % bc_f1)
+    print('c_prec:%.5f' % c_prec)
+    print('c_rec:%.5f' % c_rec)
+    print('c_f1:%.5f' % c_f1)
+    print('macro:%.5f' % macro)
+
     with open(os.path.join(output, 'scores.txt'), 'w') as fp:
         fp.write('abc_prec:%.5f\n'% abc_prec)
         fp.write('abc_rec:%.5f\n' % abc_rec)

evaluate.py score_training.pyevaluate.py renamed to score_training.py

@@ -167,13 +167,10 @@ def evaluate_links(gold_links_file, dev_links_file, sentences, gold_phrases, dev
     print("F1: %.2f" % f1)
 
 
-def evaluate(file):
-    gold = sys.argv[1] if len(sys.argv) > 1 else "gold"
-    dev = sys.argv[2] if len(sys.argv) > 2 else "dev"
-
-    input_file = abspath(join(gold, 'input_%s.txt' % file))
-    gold_phrases_file = abspath(join(gold, 'output_A_%s.txt' % file))
-    dev_phrases_file = abspath(join(dev, 'output_A_%s.txt' % file))
+def evaluate(file, folder):
+    input_file = abspath(join(folder, 'input', 'input_%s.txt' % file))
+    gold_phrases_file = abspath(join(folder, 'gold', 'output_A_%s.txt' % file))
+    dev_phrases_file = abspath(join(folder, 'dev', 'output_A_%s.txt' % file))
 
     if not exists(input_file):
         raise ValueError("Input file '%s' not found." % input_file)
@@ -186,8 +183,8 @@ def evaluate(file):
 
     l = evaluate_phrases(input_file, gold_phrases_file, dev_phrases_file)
 
-    gold_labels_file = abspath(join(gold, 'output_B_%s.txt' % file))
-    dev_labels_file = abspath(join(dev, 'output_B_%s.txt' % file))
+    gold_labels_file = abspath(join(folder, 'gold', 'output_B_%s.txt' % file))
+    dev_labels_file = abspath(join(folder, 'dev', 'output_B_%s.txt' % file))
 
     if not exists(gold_labels_file):
         raise ValueError("Gold phrases file '%s' not found." % gold_phrases_file)
@@ -197,8 +194,8 @@ def evaluate(file):
     else:
         evaluate_labels(gold_labels_file, dev_labels_file, *l)
 
-    ref_links_file = abspath(join(gold, 'output_C_%s.txt' % file))
-    eval_links_file = abspath(join(dev, 'output_C_%s.txt' % file))
+    ref_links_file = abspath(join(folder, 'gold', 'output_C_%s.txt' % file))
+    eval_links_file = abspath(join(folder, 'dev', 'output_C_%s.txt' % file))
 
     if not exists(eval_links_file):
         print("\n(!) Skipping Task C: file '%s' not found. Assuming task is not completed yet." % eval_links_file)
@@ -207,9 +204,8 @@ def evaluate(file):
 
 
 if __name__ == '__main__':
-    gold = sys.argv[1] if len(sys.argv) > 1 else "gold"
-    dev = sys.argv[2] if len(sys.argv) > 2 else "dev"
+    folder = sys.argv[1] if len(sys.argv) > 1 else "training"
 
-    for fname in os.listdir(gold):
+    for fname in os.listdir(os.path.join(folder, 'input')):
         if fname.startswith("input_"):
-            evaluate(fname[6:-4])
+            evaluate(fname[6:-4], folder)