check

Author: ydshieh

.circleci/create_circleci_config.py

@@ -154,7 +154,7 @@ def to_dict(self):
             },
             {"run": {
                 "name": "Run tests",
-                "command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n 1 {additional_flags} {' '.join(pytest_flags)} tests/models/flaubert/test_modeling_flaubert.py::FlaubertModelTest::test_batching_equivalence | tee tests_output.txt)"}
+                "command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n 1 {additional_flags} {' '.join(pytest_flags)} tests/models -k test_batching_equivalence | tee tests_output.txt)"}
             },
             {"run": {"name": "Expand to show skipped tests", "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --skip"}},
             {"run": {"name": "Failed tests: show reasons",   "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --fail"}},

tests/test_modeling_common.py

@@ -808,6 +808,13 @@ def recursive_check(batched_object, single_row_object, model_name, key):
                 self.assertFalse(
                     torch.isinf(single_row_object).any(), f"Single row output has `inf` in {model_name} for key={key}"
                 )
+                a = torch.amax(torch.abs(batched_row))
+                b = torch.amax(torch.abs(single_row_object))
+                if torch.is_floating_point(a) and torch.is_floating_point(b):
+                    if a < 1e-9 or b < 1e-9:
+                        raise ValueError("hello")
+                    # breakpoint()
+                return
                 self.assertTrue(
                     (equivalence(batched_row, single_row_object)) <= 1e-03,
                     msg=(
@@ -819,18 +826,18 @@ def recursive_check(batched_object, single_row_object, model_name, key):
         config, batched_input = self.model_tester.prepare_config_and_inputs_for_common()
         equivalence = get_tensor_equivalence_function(batched_input)
 
-        set_model_tester_for_less_flaky_test(self)
+        #set_model_tester_for_less_flaky_test(self)
 
         for model_class in self.all_model_classes:
             config.output_hidden_states = True
-            set_config_for_less_flaky_test(config)
+            #set_config_for_less_flaky_test(config)
 
             model_name = model_class.__name__
             if hasattr(self.model_tester, "prepare_config_and_inputs_for_model_class"):
                 config, batched_input = self.model_tester.prepare_config_and_inputs_for_model_class(model_class)
             batched_input_prepared = self._prepare_for_class(batched_input, model_class)
             model = model_class(config).to(torch_device).eval()
-            set_model_for_less_flaky_test(model)
+            #set_model_for_less_flaky_test(model)
 
             batch_size = self.model_tester.batch_size
             single_row_input = {}