From b8bbe29001c5a5e3b6d540525dbe5d16a96abee2 Mon Sep 17 00:00:00 2001 From: andrewtruong Date: Mon, 2 Feb 2026 12:45:19 -0500 Subject: [PATCH 1/5] test --- tests/trace/test_custom_objs.py | 82 ++++++++ .../type_handlers/test_type_handler_safety.py | 176 ++++++++++++++++++ weave/trace/serialization/custom_objs.py | 13 +- 3 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 tests/trace/type_handlers/test_type_handler_safety.py diff --git a/tests/trace/test_custom_objs.py b/tests/trace/test_custom_objs.py index 8bdec4402d99..5418814c8c4d 100644 --- a/tests/trace/test_custom_objs.py +++ b/tests/trace/test_custom_objs.py @@ -1,9 +1,13 @@ +from __future__ import annotations + +import logging from datetime import datetime, timezone import rich.markdown from PIL import Image import weave +from weave.trace.serialization import serializer from weave.trace.serialization.custom_objs import ( KNOWN_TYPES, decode_custom_obj, @@ -90,3 +94,81 @@ def make_datetime(): # due to deserializing a custom object calls = client.get_calls() assert len(calls) == 1 + + +class FailingSaveType: + """A type whose serializer save function always raises an exception.""" + + def __init__(self, value: str): + self.value = value + + def __repr__(self) -> str: + return f"FailingSaveType({self.value!r})" + + +def _failing_save(obj, artifact, name): + """A save function that always raises an exception.""" + raise RuntimeError("Intentional failure in save function") + + +def _failing_load(artifact, name, val): + """A load function (not used in these tests).""" + return FailingSaveType(val) + + +def test_encode_custom_obj_save_exception_returns_none(client, caplog): + """ + Requirement: Type handler save exceptions should not crash user code + Interface: encode_custom_obj function + Given: A serializer is registered whose save function raises an exception + When: encode_custom_obj is called with an object of that type + Then: Returns None (graceful degradation) and logs a warning + """ + # Register a serializer that will fail + serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) + + try: + obj = FailingSaveType("test_value") + + with caplog.at_level(logging.WARNING): + result = encode_custom_obj(obj) + + # Should return None instead of raising + assert result is None + + # Should log a warning about the failure + assert any("save" in record.message.lower() or "fail" in record.message.lower() + for record in caplog.records) + finally: + # Clean up: remove the registered serializer + serializer.SERIALIZERS[:] = [ + s for s in serializer.SERIALIZERS + if s.target_class is not FailingSaveType + ] + + +def test_encode_custom_obj_save_exception_does_not_propagate(client): + """ + Requirement: Type handler save exceptions must not propagate to user code + Interface: encode_custom_obj function + Given: A serializer is registered whose save function raises RuntimeError + When: encode_custom_obj is called + Then: No exception is raised to the caller + """ + # Register a serializer that will fail + serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) + + try: + obj = FailingSaveType("test_value") + + # This should NOT raise - if it does, the test fails + result = encode_custom_obj(obj) + + # We expect None as the graceful degradation + assert result is None + finally: + # Clean up: remove the registered serializer + serializer.SERIALIZERS[:] = [ + s for s in serializer.SERIALIZERS + if s.target_class is not FailingSaveType + ] diff --git a/tests/trace/type_handlers/test_type_handler_safety.py b/tests/trace/type_handlers/test_type_handler_safety.py new file mode 100644 index 000000000000..a8a0ced63fda --- /dev/null +++ b/tests/trace/type_handlers/test_type_handler_safety.py @@ -0,0 +1,176 @@ +""" +Tests to verify that type handlers never crash user code. + +Requirement: The weave op decorator is complex but should never crash user code. +Type handlers that fail during serialization should gracefully degrade without +affecting the user's program execution. +""" + +from __future__ import annotations + +import weave +from weave.trace.serialization import serializer + + +class FailingSaveType: + """A type whose serializer save function always raises an exception.""" + + def __init__(self, value: str): + self.value = value + + def __repr__(self) -> str: + return f"FailingSaveType({self.value!r})" + + +def _failing_save(obj, artifact, name): + """A save function that always raises an exception.""" + raise RuntimeError("Intentional failure in save function") + + +def _failing_load(artifact, name, val): + """A load function (not used in these tests).""" + return FailingSaveType(val) + + +def _register_failing_serializer(): + """Register the failing serializer for tests.""" + serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) + + +def _cleanup_failing_serializer(): + """Remove the failing serializer after tests.""" + serializer.SERIALIZERS[:] = [ + s for s in serializer.SERIALIZERS + if s.target_class is not FailingSaveType + ] + + +def test_op_output_with_failing_serializer_returns_value(client): + """ + Requirement: Op functions must return their values even when serialization fails + Interface: @weave.op decorated function returning an object with failing type handler + Given: An @weave.op function returns an object whose type handler save raises an exception + When: The function is called + Then: The function returns the correct value to the user (not None, not an exception) + """ + _register_failing_serializer() + + try: + @weave.op + def return_failing_type(value: str) -> FailingSaveType: + return FailingSaveType(value) + + # This should NOT raise - the user should get their return value + result = return_failing_type("hello") + + # The user must receive the actual object they created + assert isinstance(result, FailingSaveType) + assert result.value == "hello" + finally: + _cleanup_failing_serializer() + + +def test_op_input_with_failing_serializer_executes_normally(client): + """ + Requirement: Op functions must execute normally even when input serialization fails + Interface: @weave.op decorated function accepting an object with failing type handler + Given: An @weave.op function accepts an object whose type handler save raises an exception + When: The function is called + Then: The function executes normally and returns the expected result + """ + _register_failing_serializer() + + try: + @weave.op + def process_failing_type(obj: FailingSaveType) -> str: + return f"processed: {obj.value}" + + failing_obj = FailingSaveType("test_input") + + # This should NOT raise - the function should execute normally + result = process_failing_type(failing_obj) + + # The function must return its computed result + assert result == "processed: test_input" + finally: + _cleanup_failing_serializer() + + +def test_op_with_failing_serializer_call_is_recorded(client): + """ + Requirement: Calls should still be recorded even when serialization fails (with stringified fallback) + Interface: @weave.op decorated function and call record retrieval + Given: An @weave.op function returns an object whose type handler save fails + When: The function is called and we fetch the call record + Then: The call is recorded with a stringified representation of the failed object + """ + _register_failing_serializer() + + try: + @weave.op + def return_failing_for_record(value: str) -> FailingSaveType: + return FailingSaveType(value) + + # Call the function + result = return_failing_for_record("record_test") + + # Ensure the result was returned correctly + assert isinstance(result, FailingSaveType) + assert result.value == "record_test" + + # Flush to ensure the call is recorded + client.flush() + + # Get the call record + calls = return_failing_for_record.calls() + assert len(calls) == 1 + + call = calls[0] + + # The output should be recorded - either as the actual object (if serialization + # worked on the second try or there's a fallback) or as a stringified version + # The key assertion is that the call record exists and has an output + assert call.output is not None + + # If it fell back to stringify, it would be a string representation + # If serialization succeeded elsewhere, it might be the actual object + # Either way, the call should be recorded + output_str = str(call.output) + assert "record_test" in output_str or "FailingSaveType" in output_str + finally: + _cleanup_failing_serializer() + + +def test_op_with_multiple_args_one_failing_serializer(client): + """ + Requirement: A failing serializer for one argument should not affect other arguments + Interface: @weave.op decorated function with multiple arguments + Given: An @weave.op function has multiple args, one with a failing type handler + When: The function is called + Then: The function executes normally and non-failing arguments are serialized properly + """ + _register_failing_serializer() + + try: + @weave.op + def mixed_args(normal_arg: str, failing_arg: FailingSaveType) -> str: + return f"{normal_arg}: {failing_arg.value}" + + failing_obj = FailingSaveType("failing_value") + + # This should NOT raise + result = mixed_args("normal", failing_obj) + + # Function should execute normally + assert result == "normal: failing_value" + + # Verify call was recorded + client.flush() + calls = mixed_args.calls() + assert len(calls) == 1 + + call = calls[0] + # The normal_arg should be serialized properly + assert call.inputs["normal_arg"] == "normal" + finally: + _cleanup_failing_serializer() diff --git a/weave/trace/serialization/custom_objs.py b/weave/trace/serialization/custom_objs.py index 122418205dc6..0bd2ea5cb900 100644 --- a/weave/trace/serialization/custom_objs.py +++ b/weave/trace/serialization/custom_objs.py @@ -104,7 +104,18 @@ def encode_custom_obj(obj: Any) -> EncodedCustomObjDict | None: } art = MemTraceFilesArtifact() - val = serializer.save(obj, art, "obj") + try: + val = serializer.save(obj, art, "obj") + except Exception: + # Type handler save functions should never crash user code. + # If a serializer fails, we log a warning and return None, + # which will cause the caller to fall back to stringify(). + logger.warning( + f"Failed to serialize object of type {type(obj).__name__}. " + "Falling back to string representation.", + exc_info=True, + ) + return None if art.path_contents: encoded_path_contents = { k: (v.encode("utf-8") if isinstance(v, str) else v) # type: ignore From af9e3ccd2d2e414d69f621d0faef83b94657bf9d Mon Sep 17 00:00:00 2001 From: andrewtruong Date: Mon, 2 Feb 2026 12:55:34 -0500 Subject: [PATCH 2/5] test --- tests/trace/conftest.py | 38 ++++ tests/trace/test_custom_objs.py | 74 ++------ .../type_handlers/test_type_handler_safety.py | 170 ++++++------------ 3 files changed, 112 insertions(+), 170 deletions(-) create mode 100644 tests/trace/conftest.py diff --git a/tests/trace/conftest.py b/tests/trace/conftest.py new file mode 100644 index 000000000000..d0ef754a6925 --- /dev/null +++ b/tests/trace/conftest.py @@ -0,0 +1,38 @@ +"""Shared fixtures for trace tests.""" + +from __future__ import annotations + +import pytest + +from weave.trace.serialization import serializer + + +class FailingSaveType: + """A type whose serializer save function always raises an exception.""" + + def __init__(self, value: str): + self.value = value + + def __repr__(self) -> str: + return f"FailingSaveType({self.value!r})" + + +def _failing_save(obj, artifact, name): + """A save function that always raises an exception.""" + raise RuntimeError("Intentional failure in save function") + + +def _failing_load(artifact, name, val): + """A load function (not used in these tests).""" + return FailingSaveType(val) + + +@pytest.fixture +def failing_serializer(): + """Register a serializer that always fails, and clean up after the test.""" + serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) + yield FailingSaveType + serializer.SERIALIZERS[:] = [ + s for s in serializer.SERIALIZERS + if s.target_class is not FailingSaveType + ] diff --git a/tests/trace/test_custom_objs.py b/tests/trace/test_custom_objs.py index 5418814c8c4d..c76bda34ee27 100644 --- a/tests/trace/test_custom_objs.py +++ b/tests/trace/test_custom_objs.py @@ -1,19 +1,19 @@ from __future__ import annotations -import logging from datetime import datetime, timezone import rich.markdown from PIL import Image import weave -from weave.trace.serialization import serializer from weave.trace.serialization.custom_objs import ( KNOWN_TYPES, decode_custom_obj, encode_custom_obj, ) +from tests.trace.conftest import FailingSaveType + def test_encode_custom_obj_unknown_type(client): """No encoding should be done for unregistered types.""" @@ -96,58 +96,24 @@ def make_datetime(): assert len(calls) == 1 -class FailingSaveType: - """A type whose serializer save function always raises an exception.""" - - def __init__(self, value: str): - self.value = value - - def __repr__(self) -> str: - return f"FailingSaveType({self.value!r})" - - -def _failing_save(obj, artifact, name): - """A save function that always raises an exception.""" - raise RuntimeError("Intentional failure in save function") - - -def _failing_load(artifact, name, val): - """A load function (not used in these tests).""" - return FailingSaveType(val) - - -def test_encode_custom_obj_save_exception_returns_none(client, caplog): +def test_encode_custom_obj_save_exception_returns_none(client, failing_serializer): """ Requirement: Type handler save exceptions should not crash user code Interface: encode_custom_obj function Given: A serializer is registered whose save function raises an exception When: encode_custom_obj is called with an object of that type - Then: Returns None (graceful degradation) and logs a warning + Then: Returns None (graceful degradation) """ - # Register a serializer that will fail - serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) - - try: - obj = FailingSaveType("test_value") - - with caplog.at_level(logging.WARNING): - result = encode_custom_obj(obj) + obj = FailingSaveType("test_value") - # Should return None instead of raising - assert result is None + # This should NOT raise - if it does, the test fails + result = encode_custom_obj(obj) - # Should log a warning about the failure - assert any("save" in record.message.lower() or "fail" in record.message.lower() - for record in caplog.records) - finally: - # Clean up: remove the registered serializer - serializer.SERIALIZERS[:] = [ - s for s in serializer.SERIALIZERS - if s.target_class is not FailingSaveType - ] + # Should return None instead of raising + assert result is None -def test_encode_custom_obj_save_exception_does_not_propagate(client): +def test_encode_custom_obj_save_exception_does_not_propagate(client, failing_serializer): """ Requirement: Type handler save exceptions must not propagate to user code Interface: encode_custom_obj function @@ -155,20 +121,10 @@ def test_encode_custom_obj_save_exception_does_not_propagate(client): When: encode_custom_obj is called Then: No exception is raised to the caller """ - # Register a serializer that will fail - serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) - - try: - obj = FailingSaveType("test_value") + obj = FailingSaveType("test_value") - # This should NOT raise - if it does, the test fails - result = encode_custom_obj(obj) + # This should NOT raise - if it does, the test fails + result = encode_custom_obj(obj) - # We expect None as the graceful degradation - assert result is None - finally: - # Clean up: remove the registered serializer - serializer.SERIALIZERS[:] = [ - s for s in serializer.SERIALIZERS - if s.target_class is not FailingSaveType - ] + # We expect None as the graceful degradation + assert result is None diff --git a/tests/trace/type_handlers/test_type_handler_safety.py b/tests/trace/type_handlers/test_type_handler_safety.py index a8a0ced63fda..c21e98abcb5c 100644 --- a/tests/trace/type_handlers/test_type_handler_safety.py +++ b/tests/trace/type_handlers/test_type_handler_safety.py @@ -9,43 +9,11 @@ from __future__ import annotations import weave -from weave.trace.serialization import serializer +from tests.trace.conftest import FailingSaveType -class FailingSaveType: - """A type whose serializer save function always raises an exception.""" - def __init__(self, value: str): - self.value = value - - def __repr__(self) -> str: - return f"FailingSaveType({self.value!r})" - - -def _failing_save(obj, artifact, name): - """A save function that always raises an exception.""" - raise RuntimeError("Intentional failure in save function") - - -def _failing_load(artifact, name, val): - """A load function (not used in these tests).""" - return FailingSaveType(val) - - -def _register_failing_serializer(): - """Register the failing serializer for tests.""" - serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) - - -def _cleanup_failing_serializer(): - """Remove the failing serializer after tests.""" - serializer.SERIALIZERS[:] = [ - s for s in serializer.SERIALIZERS - if s.target_class is not FailingSaveType - ] - - -def test_op_output_with_failing_serializer_returns_value(client): +def test_op_output_with_failing_serializer_returns_value(client, failing_serializer): """ Requirement: Op functions must return their values even when serialization fails Interface: @weave.op decorated function returning an object with failing type handler @@ -53,24 +21,19 @@ def test_op_output_with_failing_serializer_returns_value(client): When: The function is called Then: The function returns the correct value to the user (not None, not an exception) """ - _register_failing_serializer() - - try: - @weave.op - def return_failing_type(value: str) -> FailingSaveType: - return FailingSaveType(value) + @weave.op + def return_failing_type(value: str) -> FailingSaveType: + return FailingSaveType(value) - # This should NOT raise - the user should get their return value - result = return_failing_type("hello") + # This should NOT raise - the user should get their return value + result = return_failing_type("hello") - # The user must receive the actual object they created - assert isinstance(result, FailingSaveType) - assert result.value == "hello" - finally: - _cleanup_failing_serializer() + # The user must receive the actual object they created + assert isinstance(result, FailingSaveType) + assert result.value == "hello" -def test_op_input_with_failing_serializer_executes_normally(client): +def test_op_input_with_failing_serializer_executes_normally(client, failing_serializer): """ Requirement: Op functions must execute normally even when input serialization fails Interface: @weave.op decorated function accepting an object with failing type handler @@ -78,25 +41,20 @@ def test_op_input_with_failing_serializer_executes_normally(client): When: The function is called Then: The function executes normally and returns the expected result """ - _register_failing_serializer() + @weave.op + def process_failing_type(obj: FailingSaveType) -> str: + return f"processed: {obj.value}" - try: - @weave.op - def process_failing_type(obj: FailingSaveType) -> str: - return f"processed: {obj.value}" + failing_obj = FailingSaveType("test_input") - failing_obj = FailingSaveType("test_input") + # This should NOT raise - the function should execute normally + result = process_failing_type(failing_obj) - # This should NOT raise - the function should execute normally - result = process_failing_type(failing_obj) + # The function must return its computed result + assert result == "processed: test_input" - # The function must return its computed result - assert result == "processed: test_input" - finally: - _cleanup_failing_serializer() - -def test_op_with_failing_serializer_call_is_recorded(client): +def test_op_with_failing_serializer_call_is_recorded(client, failing_serializer): """ Requirement: Calls should still be recorded even when serialization fails (with stringified fallback) Interface: @weave.op decorated function and call record retrieval @@ -104,44 +62,39 @@ def test_op_with_failing_serializer_call_is_recorded(client): When: The function is called and we fetch the call record Then: The call is recorded with a stringified representation of the failed object """ - _register_failing_serializer() - - try: - @weave.op - def return_failing_for_record(value: str) -> FailingSaveType: - return FailingSaveType(value) + @weave.op + def return_failing_for_record(value: str) -> FailingSaveType: + return FailingSaveType(value) - # Call the function - result = return_failing_for_record("record_test") + # Call the function + result = return_failing_for_record("record_test") - # Ensure the result was returned correctly - assert isinstance(result, FailingSaveType) - assert result.value == "record_test" + # Ensure the result was returned correctly + assert isinstance(result, FailingSaveType) + assert result.value == "record_test" - # Flush to ensure the call is recorded - client.flush() + # Flush to ensure the call is recorded + client.flush() - # Get the call record - calls = return_failing_for_record.calls() - assert len(calls) == 1 + # Get the call record + calls = return_failing_for_record.calls() + assert len(calls) == 1 - call = calls[0] + call = calls[0] - # The output should be recorded - either as the actual object (if serialization - # worked on the second try or there's a fallback) or as a stringified version - # The key assertion is that the call record exists and has an output - assert call.output is not None + # The output should be recorded - either as the actual object (if serialization + # worked on the second try or there's a fallback) or as a stringified version + # The key assertion is that the call record exists and has an output + assert call.output is not None - # If it fell back to stringify, it would be a string representation - # If serialization succeeded elsewhere, it might be the actual object - # Either way, the call should be recorded - output_str = str(call.output) - assert "record_test" in output_str or "FailingSaveType" in output_str - finally: - _cleanup_failing_serializer() + # If it fell back to stringify, it would be a string representation + # If serialization succeeded elsewhere, it might be the actual object + # Either way, the call should be recorded + output_str = str(call.output) + assert "record_test" in output_str or "FailingSaveType" in output_str -def test_op_with_multiple_args_one_failing_serializer(client): +def test_op_with_multiple_args_one_failing_serializer(client, failing_serializer): """ Requirement: A failing serializer for one argument should not affect other arguments Interface: @weave.op decorated function with multiple arguments @@ -149,28 +102,23 @@ def test_op_with_multiple_args_one_failing_serializer(client): When: The function is called Then: The function executes normally and non-failing arguments are serialized properly """ - _register_failing_serializer() - - try: - @weave.op - def mixed_args(normal_arg: str, failing_arg: FailingSaveType) -> str: - return f"{normal_arg}: {failing_arg.value}" + @weave.op + def mixed_args(normal_arg: str, failing_arg: FailingSaveType) -> str: + return f"{normal_arg}: {failing_arg.value}" - failing_obj = FailingSaveType("failing_value") + failing_obj = FailingSaveType("failing_value") - # This should NOT raise - result = mixed_args("normal", failing_obj) + # This should NOT raise + result = mixed_args("normal", failing_obj) - # Function should execute normally - assert result == "normal: failing_value" + # Function should execute normally + assert result == "normal: failing_value" - # Verify call was recorded - client.flush() - calls = mixed_args.calls() - assert len(calls) == 1 + # Verify call was recorded + client.flush() + calls = mixed_args.calls() + assert len(calls) == 1 - call = calls[0] - # The normal_arg should be serialized properly - assert call.inputs["normal_arg"] == "normal" - finally: - _cleanup_failing_serializer() + call = calls[0] + # The normal_arg should be serialized properly + assert call.inputs["normal_arg"] == "normal" From 8cc0dfdd0012f2f483817564db885e13db17d225 Mon Sep 17 00:00:00 2001 From: andrewtruong Date: Mon, 2 Feb 2026 12:56:52 -0500 Subject: [PATCH 3/5] test --- tests/trace/conftest.py | 22 ++---------------- tests/trace/test_custom_objs.py | 2 +- tests/trace/test_utils.py | 23 +++++++++++++++++++ .../type_handlers/test_type_handler_safety.py | 2 +- 4 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 tests/trace/test_utils.py diff --git a/tests/trace/conftest.py b/tests/trace/conftest.py index d0ef754a6925..cc885f4e2dd8 100644 --- a/tests/trace/conftest.py +++ b/tests/trace/conftest.py @@ -6,31 +6,13 @@ from weave.trace.serialization import serializer - -class FailingSaveType: - """A type whose serializer save function always raises an exception.""" - - def __init__(self, value: str): - self.value = value - - def __repr__(self) -> str: - return f"FailingSaveType({self.value!r})" - - -def _failing_save(obj, artifact, name): - """A save function that always raises an exception.""" - raise RuntimeError("Intentional failure in save function") - - -def _failing_load(artifact, name, val): - """A load function (not used in these tests).""" - return FailingSaveType(val) +from tests.trace.test_utils import FailingSaveType, failing_load, failing_save @pytest.fixture def failing_serializer(): """Register a serializer that always fails, and clean up after the test.""" - serializer.register_serializer(FailingSaveType, _failing_save, _failing_load) + serializer.register_serializer(FailingSaveType, failing_save, failing_load) yield FailingSaveType serializer.SERIALIZERS[:] = [ s for s in serializer.SERIALIZERS diff --git a/tests/trace/test_custom_objs.py b/tests/trace/test_custom_objs.py index c76bda34ee27..d2087209a4d5 100644 --- a/tests/trace/test_custom_objs.py +++ b/tests/trace/test_custom_objs.py @@ -12,7 +12,7 @@ encode_custom_obj, ) -from tests.trace.conftest import FailingSaveType +from tests.trace.test_utils import FailingSaveType def test_encode_custom_obj_unknown_type(client): diff --git a/tests/trace/test_utils.py b/tests/trace/test_utils.py new file mode 100644 index 000000000000..e06e504d441c --- /dev/null +++ b/tests/trace/test_utils.py @@ -0,0 +1,23 @@ +"""Shared test utilities for trace tests.""" + +from __future__ import annotations + + +class FailingSaveType: + """A type whose serializer save function always raises an exception.""" + + def __init__(self, value: str): + self.value = value + + def __repr__(self) -> str: + return f"FailingSaveType({self.value!r})" + + +def failing_save(obj, artifact, name): + """A save function that always raises an exception.""" + raise RuntimeError("Intentional failure in save function") + + +def failing_load(artifact, name, val): + """A load function (not used in these tests).""" + return FailingSaveType(val) diff --git a/tests/trace/type_handlers/test_type_handler_safety.py b/tests/trace/type_handlers/test_type_handler_safety.py index c21e98abcb5c..489246dd025d 100644 --- a/tests/trace/type_handlers/test_type_handler_safety.py +++ b/tests/trace/type_handlers/test_type_handler_safety.py @@ -10,7 +10,7 @@ import weave -from tests.trace.conftest import FailingSaveType +from tests.trace.test_utils import FailingSaveType def test_op_output_with_failing_serializer_returns_value(client, failing_serializer): From b38e4520bee4ada99a1ce23f4214e5a27e1a36f3 Mon Sep 17 00:00:00 2001 From: andrewtruong Date: Mon, 2 Feb 2026 14:00:40 -0500 Subject: [PATCH 4/5] test --- tests/trace/conftest.py | 6 ++---- tests/trace/test_custom_objs.py | 13 ++++++------ .../type_handlers/test_type_handler_safety.py | 20 +++++++++---------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/tests/trace/conftest.py b/tests/trace/conftest.py index cc885f4e2dd8..864981b59765 100644 --- a/tests/trace/conftest.py +++ b/tests/trace/conftest.py @@ -4,9 +4,8 @@ import pytest -from weave.trace.serialization import serializer - from tests.trace.test_utils import FailingSaveType, failing_load, failing_save +from weave.trace.serialization import serializer @pytest.fixture @@ -15,6 +14,5 @@ def failing_serializer(): serializer.register_serializer(FailingSaveType, failing_save, failing_load) yield FailingSaveType serializer.SERIALIZERS[:] = [ - s for s in serializer.SERIALIZERS - if s.target_class is not FailingSaveType + s for s in serializer.SERIALIZERS if s.target_class is not FailingSaveType ] diff --git a/tests/trace/test_custom_objs.py b/tests/trace/test_custom_objs.py index d2087209a4d5..13f418231935 100644 --- a/tests/trace/test_custom_objs.py +++ b/tests/trace/test_custom_objs.py @@ -6,14 +6,13 @@ from PIL import Image import weave +from tests.trace.test_utils import FailingSaveType from weave.trace.serialization.custom_objs import ( KNOWN_TYPES, decode_custom_obj, encode_custom_obj, ) -from tests.trace.test_utils import FailingSaveType - def test_encode_custom_obj_unknown_type(client): """No encoding should be done for unregistered types.""" @@ -97,8 +96,7 @@ def make_datetime(): def test_encode_custom_obj_save_exception_returns_none(client, failing_serializer): - """ - Requirement: Type handler save exceptions should not crash user code + """Requirement: Type handler save exceptions should not crash user code Interface: encode_custom_obj function Given: A serializer is registered whose save function raises an exception When: encode_custom_obj is called with an object of that type @@ -113,9 +111,10 @@ def test_encode_custom_obj_save_exception_returns_none(client, failing_serialize assert result is None -def test_encode_custom_obj_save_exception_does_not_propagate(client, failing_serializer): - """ - Requirement: Type handler save exceptions must not propagate to user code +def test_encode_custom_obj_save_exception_does_not_propagate( + client, failing_serializer +): + """Requirement: Type handler save exceptions must not propagate to user code Interface: encode_custom_obj function Given: A serializer is registered whose save function raises RuntimeError When: encode_custom_obj is called diff --git a/tests/trace/type_handlers/test_type_handler_safety.py b/tests/trace/type_handlers/test_type_handler_safety.py index 489246dd025d..332cfd4c1cec 100644 --- a/tests/trace/type_handlers/test_type_handler_safety.py +++ b/tests/trace/type_handlers/test_type_handler_safety.py @@ -1,5 +1,4 @@ -""" -Tests to verify that type handlers never crash user code. +"""Tests to verify that type handlers never crash user code. Requirement: The weave op decorator is complex but should never crash user code. Type handlers that fail during serialization should gracefully degrade without @@ -9,18 +8,17 @@ from __future__ import annotations import weave - from tests.trace.test_utils import FailingSaveType def test_op_output_with_failing_serializer_returns_value(client, failing_serializer): - """ - Requirement: Op functions must return their values even when serialization fails + """Requirement: Op functions must return their values even when serialization fails Interface: @weave.op decorated function returning an object with failing type handler Given: An @weave.op function returns an object whose type handler save raises an exception When: The function is called Then: The function returns the correct value to the user (not None, not an exception) """ + @weave.op def return_failing_type(value: str) -> FailingSaveType: return FailingSaveType(value) @@ -34,13 +32,13 @@ def return_failing_type(value: str) -> FailingSaveType: def test_op_input_with_failing_serializer_executes_normally(client, failing_serializer): - """ - Requirement: Op functions must execute normally even when input serialization fails + """Requirement: Op functions must execute normally even when input serialization fails Interface: @weave.op decorated function accepting an object with failing type handler Given: An @weave.op function accepts an object whose type handler save raises an exception When: The function is called Then: The function executes normally and returns the expected result """ + @weave.op def process_failing_type(obj: FailingSaveType) -> str: return f"processed: {obj.value}" @@ -55,13 +53,13 @@ def process_failing_type(obj: FailingSaveType) -> str: def test_op_with_failing_serializer_call_is_recorded(client, failing_serializer): - """ - Requirement: Calls should still be recorded even when serialization fails (with stringified fallback) + """Requirement: Calls should still be recorded even when serialization fails (with stringified fallback) Interface: @weave.op decorated function and call record retrieval Given: An @weave.op function returns an object whose type handler save fails When: The function is called and we fetch the call record Then: The call is recorded with a stringified representation of the failed object """ + @weave.op def return_failing_for_record(value: str) -> FailingSaveType: return FailingSaveType(value) @@ -95,13 +93,13 @@ def return_failing_for_record(value: str) -> FailingSaveType: def test_op_with_multiple_args_one_failing_serializer(client, failing_serializer): - """ - Requirement: A failing serializer for one argument should not affect other arguments + """Requirement: A failing serializer for one argument should not affect other arguments Interface: @weave.op decorated function with multiple arguments Given: An @weave.op function has multiple args, one with a failing type handler When: The function is called Then: The function executes normally and non-failing arguments are serialized properly """ + @weave.op def mixed_args(normal_arg: str, failing_arg: FailingSaveType) -> str: return f"{normal_arg}: {failing_arg.value}" From 2fc93f5351d3b1f0cd790583afa64896a8538ec6 Mon Sep 17 00:00:00 2001 From: andrewtruong Date: Tue, 3 Feb 2026 10:39:22 -0500 Subject: [PATCH 5/5] test --- .../type_handlers/test_type_handler_safety.py | 66 ++++++++++--------- weave/trace/serialization/custom_objs.py | 3 + 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/tests/trace/type_handlers/test_type_handler_safety.py b/tests/trace/type_handlers/test_type_handler_safety.py index 332cfd4c1cec..89395fc31e5f 100644 --- a/tests/trace/type_handlers/test_type_handler_safety.py +++ b/tests/trace/type_handlers/test_type_handler_safety.py @@ -11,7 +11,7 @@ from tests.trace.test_utils import FailingSaveType -def test_op_output_with_failing_serializer_returns_value(client, failing_serializer): +def test_op_output_with_failing_serializer_does_not_raise(client, failing_serializer): """Requirement: Op functions must return their values even when serialization fails Interface: @weave.op decorated function returning an object with failing type handler Given: An @weave.op function returns an object whose type handler save raises an exception @@ -31,7 +31,7 @@ def return_failing_type(value: str) -> FailingSaveType: assert result.value == "hello" -def test_op_input_with_failing_serializer_executes_normally(client, failing_serializer): +def test_op_input_with_failing_serializer_does_not_raise(client, failing_serializer): """Requirement: Op functions must execute normally even when input serialization fails Interface: @weave.op decorated function accepting an object with failing type handler Given: An @weave.op function accepts an object whose type handler save raises an exception @@ -52,6 +52,38 @@ def process_failing_type(obj: FailingSaveType) -> str: assert result == "processed: test_input" +def test_op_with_multiple_args_one_failing_serializer_does_not_raise( + client, failing_serializer +): + """Requirement: A failing serializer for one argument should not affect other arguments + Interface: @weave.op decorated function with multiple arguments + Given: An @weave.op function has multiple args, one with a failing type handler + When: The function is called + Then: The function executes normally and non-failing arguments are serialized properly + """ + + @weave.op + def mixed_args(normal_arg: str, failing_arg: FailingSaveType) -> str: + return f"{normal_arg}: {failing_arg.value}" + + failing_obj = FailingSaveType("failing_value") + + # This should NOT raise + result = mixed_args("normal", failing_obj) + + # Function should execute normally + assert result == "normal: failing_value" + + # Verify call was recorded + client.flush() + calls = mixed_args.calls() + assert len(calls) == 1 + + call = calls[0] + # The normal_arg should be serialized properly + assert call.inputs["normal_arg"] == "normal" + + def test_op_with_failing_serializer_call_is_recorded(client, failing_serializer): """Requirement: Calls should still be recorded even when serialization fails (with stringified fallback) Interface: @weave.op decorated function and call record retrieval @@ -90,33 +122,3 @@ def return_failing_for_record(value: str) -> FailingSaveType: # Either way, the call should be recorded output_str = str(call.output) assert "record_test" in output_str or "FailingSaveType" in output_str - - -def test_op_with_multiple_args_one_failing_serializer(client, failing_serializer): - """Requirement: A failing serializer for one argument should not affect other arguments - Interface: @weave.op decorated function with multiple arguments - Given: An @weave.op function has multiple args, one with a failing type handler - When: The function is called - Then: The function executes normally and non-failing arguments are serialized properly - """ - - @weave.op - def mixed_args(normal_arg: str, failing_arg: FailingSaveType) -> str: - return f"{normal_arg}: {failing_arg.value}" - - failing_obj = FailingSaveType("failing_value") - - # This should NOT raise - result = mixed_args("normal", failing_obj) - - # Function should execute normally - assert result == "normal: failing_value" - - # Verify call was recorded - client.flush() - calls = mixed_args.calls() - assert len(calls) == 1 - - call = calls[0] - # The normal_arg should be serialized properly - assert call.inputs["normal_arg"] == "normal" diff --git a/weave/trace/serialization/custom_objs.py b/weave/trace/serialization/custom_objs.py index 0bd2ea5cb900..a737aa15fbdf 100644 --- a/weave/trace/serialization/custom_objs.py +++ b/weave/trace/serialization/custom_objs.py @@ -106,6 +106,9 @@ def encode_custom_obj(obj: Any) -> EncodedCustomObjDict | None: art = MemTraceFilesArtifact() try: val = serializer.save(obj, art, "obj") + # TODO: In future, this should raise a specific WeaveException that can be caught + # and managed. A higher level handler will then catch that exception and ignore + # it by default, leading to the current behaviour. except Exception: # Type handler save functions should never crash user code. # If a serializer fails, we log a warning and return None,