Fix default scoring

Author: adrien1018

.gitignore

@@ -1,4 +1,5 @@
 *.o
 *.swp
+*.swo
 build/
-.vscode/
+.vscode/

src/server_io.h

@@ -4,7 +4,7 @@
 #include <string>
 #include "tioj/submission.h"
 
-const std::string kVersionCode = "2.0.1";
+const std::string kVersionCode = "2.0.2";
 extern std::string kTIOJUrl;
 extern std::string kTIOJKey;
 extern size_t kMaxQueue;

test/default_scoring_test.cpp

@@ -0,0 +1,88 @@
+#include <array>
+#include <gtest/gtest-matchers.h>
+#include "example_problem.h"
+#include "utils.h"
+
+namespace {
+
+constexpr long kTime = 1655000000;
+
+struct ScoringParam {
+  std::vector<std::string> default_scoring_args;
+  std::array<Verdict, 10> verdicts;
+};
+
+std::string ParamName(const ::testing::TestParamInfo<ScoringParam>& info) {
+  std::string ret;
+  for (auto& i : info.param.default_scoring_args) ret += i + '_';
+  for (auto& i : ret) {
+    if (i == '-') i = '_';
+  }
+  if (ret.empty()) {
+    ret = "empty";
+  } else {
+    ret.pop_back();
+  }
+  return ret;
+}
+
+}
+
+class ExampleProblemDefaultScoring : public ExampleProblem, public testing::WithParamInterface<ScoringParam> {};
+TEST_P(ExampleProblemDefaultScoring, Mode) {
+  auto& param = GetParam();
+  SetUp(2, {
+    {"123\n234\n", "123\n234\n"}, // match: AC for all
+    {"123\n234", "123 \n234 \n\n"}, // trailing whitespaces / blank lines: AC except strict
+    {" 123 234\n", "123 234\n"}, // preceding whitespaces: AC for white-diff/float-diff
+    {"123  234\n", "123 234\n"}, // additional whitespaces: AC for white-diff/float-diff
+    {" 1.0000001  1.0000001 ", "1.0 1.0\n"}, // AC for float-diff >1e-7
+    {" 1.0000001  1.0000001 \n a 1.0000001 -1", "1.0 1.0\na 1.0 -1\n"}, // AC for float-diff >1e-7
+    {" 1.00001  1.00001 ", "1.0 1.0\n"}, // AC for float-diff >1e-5
+    {" 100.00001  100.00001 ", "100.0 100.0\n"}, // AC for float-diff (absolute>1e-5) (absolute-relative/relative>1e-7)
+    {" 100.00001 0.0100001\n", "100.0 0.01\n"}, // AC for float-diff (absolute-relative>1e-7) (absolute/relative>1e-5)
+    {"1.0000001  1.0000001\n", "1.0 1\n"}, // unmatched on non-float: WA for all
+  }, 2);
+  AssertVerdictReporter reporter(Verdict::WA, true, false);
+  sub.reporter = reporter.GetReporter();
+  auto orig_score = sub.reporter.ReportScoringResult;
+  sub.reporter.ReportScoringResult = [&](const Submission& sub, const SubmissionResult& res, int subtask, int stage) {
+    orig_score(sub, res, subtask, stage);
+    ASSERT_EQ(res.td_results[subtask].verdict, param.verdicts[subtask]) << "Subtask " << subtask;
+  };
+  sub.default_scoring_args = param.default_scoring_args;
+  long id = SetupSubmission(sub, 5, Compiler::GCC_CPP_17, kTime, true, R"(#include <unistd.h>
+char buf[256];
+int main(){ write(1, buf, read(0, buf, 256)); })", SpecjudgeType::NORMAL);
+  RunAndTeardownSubmission(id);
+}
+
+constexpr Verdict AC = Verdict::AC;
+constexpr Verdict WA = Verdict::WA;
+
+INSTANTIATE_TEST_SUITE_P(OneSubmission, ExampleProblemDefaultScoring,
+    testing::Values(
+      (ScoringParam){{"strict"},
+        {{AC, WA, WA, WA, WA, WA, WA, WA, WA, WA}}},
+      (ScoringParam){{"line"},
+        {{AC, AC, WA, WA, WA, WA, WA, WA, WA, WA}}},
+      (ScoringParam){{},
+        {{AC, AC, WA, WA, WA, WA, WA, WA, WA, WA}}},
+      (ScoringParam){{"white-diff"},
+        {{AC, AC, AC, AC, WA, WA, WA, WA, WA, WA}}},
+      (ScoringParam){{"float-diff", "absolute-relative", "1e-9"},
+        {{AC, AC, AC, AC, WA, WA, WA, WA, WA, WA}}},
+      (ScoringParam){{"float-diff", "absolute", "1e-6"},
+        {{AC, AC, AC, AC, AC, AC, WA, WA, WA, WA}}},
+      (ScoringParam){{"float-diff", "absolute"},
+        {{AC, AC, AC, AC, AC, AC, WA, WA, WA, WA}}},
+      (ScoringParam){{"float-diff", "relative"},
+        {{AC, AC, AC, AC, AC, AC, WA, AC, WA, WA}}},
+      (ScoringParam){{"float-diff", "absolute-relative"},
+        {{AC, AC, AC, AC, AC, AC, WA, AC, AC, WA}}},
+      (ScoringParam){{"float-diff"},
+        {{AC, AC, AC, AC, AC, AC, WA, AC, AC, WA}}},
+      (ScoringParam){{"float-diff", "relative", "1e-4"},
+        {{AC, AC, AC, AC, AC, AC, AC, AC, AC, WA}}}
+    ),
+    ParamName);

test/example_problem.cpp

@@ -8,6 +8,12 @@
 #include "utils.h"
 
 void ExampleProblem::SetUp(int problem_id_, int td_num, int max_parallel) {
+  std::vector<std::pair<std::string, std::string>> tds;
+  for (int i = 0; i < td_num; i++) tds.emplace_back(std::to_string(i) + '\n', std::to_string(i) + '\n');
+  SetUp(problem_id_, tds, max_parallel);
+}
+
+void ExampleProblem::SetUp(int problem_id_, const std::vector<std::pair<std::string, std::string>>& tds, int max_parallel) {
   kMaxParallel = max_parallel;
   {
     char td_path_tmp[256] = "/tmp/td_test_XXXXXX";
@@ -16,12 +22,12 @@ void ExampleProblem::SetUp(int problem_id_, int td_num, int max_parallel) {
   }
   problem_id = problem_id_;
   fs::create_directories(td_path);
-  for (int i = 0; i < td_num; i++) {
-    std::filesystem::path in_path = td_path / (std::to_string(td_num) + ".in");
-    std::filesystem::path out_path = td_path / (std::to_string(td_num) + ".out");
+  for (int i = 0; i < (int)tds.size(); i++) {
+    std::filesystem::path in_path = td_path / (std::to_string(i) + ".in");
+    std::filesystem::path out_path = td_path / (std::to_string(i) + ".out");
     std::ofstream f1(in_path), f2(out_path);
-    f1 << i << std::endl;
-    f2 << i << std::endl;
+    f1 << tds[i].first;
+    f2 << tds[i].second;
     sub.testdata.push_back({in_path, out_path, 65536, 65536, 65536, 1'000'000, false});
   }
   sub.problem_id = problem_id;

test/example_problem.h

@@ -9,6 +9,7 @@
 class ExampleProblem : public ::testing::Test {
  protected:
   void SetUp(int problem_id_, int td_num, int max_parallel = 1);
+  void SetUp(int problem_id_, const std::vector<std::pair<std::string, std::string>>& tds, int max_parallel = 1);
   void TearDown() override;
 
   void RunAndTeardownSubmission(long id);

test/utils.h

@@ -9,17 +9,19 @@ class AssertVerdictReporter {
   bool has_scoring_result_;
   bool has_overall_result_;
   bool require_scoring_;
+  bool assert_scoring_verdict_;
 
   void AssertResult_() {
     ASSERT_TRUE(has_overall_result_ && (!require_scoring_ || has_scoring_result_));
   }
  public:
   Verdict expect_verdict;
 
-  AssertVerdictReporter(Verdict ver, bool require_scoring = true) :
+  AssertVerdictReporter(Verdict ver, bool require_scoring = true, bool assert_scoring_verdict = true) :
       has_scoring_result_(false),
       has_overall_result_(false),
       require_scoring_(require_scoring),
+      assert_scoring_verdict_(assert_scoring_verdict),
       expect_verdict(ver) {}
   ~AssertVerdictReporter() { AssertResult_(); }
 
@@ -28,7 +30,9 @@ class AssertVerdictReporter {
     has_overall_result_ = true;
   }
   void ReportScoringResult(const SubmissionResult& res, int subtask) {
-    ASSERT_EQ(res.td_results[subtask].verdict, expect_verdict);
+    if (assert_scoring_verdict_) {
+      ASSERT_EQ(res.td_results[subtask].verdict, expect_verdict);
+    }
     has_scoring_result_ = true;
   }
 

tools/default-scoring.cpp

@@ -170,7 +170,7 @@ int main(int argc, char** argv) {
   }
 
   // parse arguments
-  std::string type = "line", subtype;
+  std::string type = "line", subtype = "absolute-relative";
   double threshold = 1e-6;
   {
     for (int i = 2; i < argc; i++) {
@@ -184,7 +184,7 @@ int main(int argc, char** argv) {
         subtype = argv[++i];
         if (i + 1 < argc) {
           try {
-            threshold = std::stold(std::string(argv[i]));
+            threshold = std::stold(std::string(argv[i + 1]));
             i++;
           } catch (...) {}
         }