feat(python): add smoke test generation for production environment testing

generators/python/core_utilities/sdk/smoke_test_utilities.py

@@ -0,0 +1,225 @@
+"""
+Utilities for smoke tests that validate response structure and types
+without requiring exact value matches.
+"""
+
+import datetime
+import typing
+import uuid
+
+import pydantic
+
+
+def get_type_name(value: typing.Any) -> str:
+    """Get a human-readable type name for a value."""
+    if value is None:
+        return "None"
+    if isinstance(value, bool):
+        return "bool"
+    if isinstance(value, int):
+        return "int"
+    if isinstance(value, float):
+        return "float"
+    if isinstance(value, str):
+        return "str"
+    if isinstance(value, list):
+        return "list"
+    if isinstance(value, dict):
+        return "dict"
+    if isinstance(value, datetime.datetime):
+        return "datetime"
+    if isinstance(value, datetime.date):
+        return "date"
+    if isinstance(value, uuid.UUID):
+        return "uuid"
+    return type(value).__name__
+
+
+def validate_type_match(
+    actual_value: typing.Any,
+    expected_type: typing.Any,
+    field_path: str = "",
+) -> None:
+    """
+    Validate that the actual value matches the expected type.
+    Does not compare exact values, only validates the type/structure.
+    """
+    if expected_type == "no_validate":
+        return
+
+    if expected_type is None:
+        # No type expectation, just ensure the value exists
+        return
+
+    if expected_type == "uuid":
+        assert isinstance(actual_value, (str, uuid.UUID)), (
+            f"Field '{field_path}': Expected UUID, got {get_type_name(actual_value)}"
+        )
+        if isinstance(actual_value, str):
+            try:
+                uuid.UUID(actual_value)
+            except ValueError:
+                raise AssertionError(f"Field '{field_path}': Expected valid UUID string, got '{actual_value}'")
+    elif expected_type == "date":
+        assert isinstance(actual_value, (str, datetime.date)), (
+            f"Field '{field_path}': Expected date, got {get_type_name(actual_value)}"
+        )
+    elif expected_type == "datetime":
+        assert isinstance(actual_value, (str, datetime.datetime)), (
+            f"Field '{field_path}': Expected datetime, got {get_type_name(actual_value)}"
+        )
+    elif expected_type == "integer":
+        assert isinstance(actual_value, int), (
+            f"Field '{field_path}': Expected integer, got {get_type_name(actual_value)}"
+        )
+    elif expected_type == "string":
+        assert isinstance(actual_value, str), (
+            f"Field '{field_path}': Expected string, got {get_type_name(actual_value)}"
+        )
+    elif expected_type == "boolean":
+        assert isinstance(actual_value, bool), (
+            f"Field '{field_path}': Expected boolean, got {get_type_name(actual_value)}"
+        )
+    elif expected_type == "float":
+        assert isinstance(actual_value, (int, float)), (
+            f"Field '{field_path}': Expected float, got {get_type_name(actual_value)}"
+        )
+
+
+def validate_structure(
+    response: typing.Any,
+    type_expectations: typing.Any,
+    field_path: str = "",
+) -> None:
+    """
+    Validate that the response matches the expected structure and types.
+    This is a structural validation that checks:
+    - All expected fields are present
+    - Field types match the expected types
+    - Nested structures are validated recursively
+
+    Unlike strict validation, this does NOT compare exact values.
+    """
+    if type_expectations == "no_validate":
+        return
+
+    # Handle None response
+    if response is None:
+        # None is acceptable if we don't have strict type expectations
+        return
+
+    # Handle primitive types
+    if (
+        not isinstance(response, list)
+        and not isinstance(response, dict)
+        and not (hasattr(pydantic, "BaseModel") and isinstance(response, pydantic.BaseModel))
+    ):
+        validate_type_match(response, type_expectations, field_path)
+        return
+
+    # Handle lists
+    if isinstance(response, list):
+        if isinstance(type_expectations, tuple):
+            container_type = type_expectations[0]
+            contents_expectations = type_expectations[1]
+
+            # Validate it's a list (or set)
+            if container_type == "set":
+                # Sets are represented as lists in JSON
+                pass
+
+            # Validate each item in the list
+            if isinstance(contents_expectations, dict):
+                for idx, item in enumerate(response):
+                    item_path = f"{field_path}[{idx}]"
+                    item_expectation = contents_expectations.get(idx)
+                    if isinstance(item_expectation, dict):
+                        validate_structure(item, item_expectation, item_path)
+                    else:
+                        validate_type_match(item, item_expectation, item_path)
+        return
+
+    # Handle dicts and Pydantic models
+    response_dict = response
+    if hasattr(pydantic, "BaseModel") and isinstance(response, pydantic.BaseModel):
+        response_dict = response.dict(by_alias=True)
+
+    if not isinstance(response_dict, dict):
+        return
+
+    if not isinstance(type_expectations, dict):
+        return
+
+    # Validate each expected field exists and has the correct type
+    for key, expected_type in type_expectations.items():
+        current_path = f"{field_path}.{key}" if field_path else key
+
+        # Check field exists
+        assert key in response_dict, (
+            f"Field '{current_path}' not found in response. Available fields: {list(response_dict.keys())}"
+        )
+
+        actual_value = response_dict[key]
+
+        # Recursively validate nested structures
+        if isinstance(expected_type, dict):
+            validate_structure(actual_value, expected_type, current_path)
+        elif isinstance(expected_type, tuple):
+            # Container type (list, set, dict)
+            validate_structure(actual_value, expected_type, current_path)
+        else:
+            validate_type_match(actual_value, expected_type, current_path)
+
+
+def validate_smoke_test_response(
+    response: typing.Any,
+    type_expectations: typing.Any,
+) -> None:
+    """
+    Main entry point for smoke test structural validation.
+    Validates that the response matches the expected structure and types.
+
+    Args:
+        response: The actual response from the API
+        type_expectations: A nested structure describing expected types
+    """
+    validate_structure(response, type_expectations, "")
+
+
+def _normalize_value(value: typing.Any) -> typing.Any:
+    """Normalize a value for comparison (convert Pydantic models to dicts, etc.)."""
+    if value is None:
+        return None
+    if hasattr(pydantic, "BaseModel") and isinstance(value, pydantic.BaseModel):
+        return value.dict(by_alias=True)
+    if isinstance(value, list):
+        return [_normalize_value(item) for item in value]
+    if isinstance(value, dict):
+        return {k: _normalize_value(v) for k, v in value.items()}
+    if isinstance(value, datetime.datetime):
+        return value.isoformat()
+    if isinstance(value, datetime.date):
+        return value.isoformat()
+    if isinstance(value, uuid.UUID):
+        return str(value)
+    return value
+
+
+def validate_strict_response(
+    response: typing.Any,
+    expected_response: typing.Any,
+) -> None:
+    """
+    Main entry point for smoke test strict validation.
+    Validates that the response matches the expected values exactly.
+
+    Args:
+        response: The actual response from the API
+        expected_response: The expected response values (from the example)
+    """
+    normalized_response = _normalize_value(response)
+    normalized_expected = _normalize_value(expected_response)
+
+    assert normalized_response == normalized_expected, (
+        f"Response does not match expected values.\nExpected: {normalized_expected}\nActual: {normalized_response}"
+    )

generators/python/sdk/versions.yml

@@ -1,6 +1,19 @@
 # yaml-language-server: $schema=../../../fern-versions-yml.schema.json
 # For unreleased changes, use unreleased.yml
 
+- version: 4.47.0-rc0
+  changelogEntry:
+    - summary: |
+        Add smoke test generation feature with layered validation modes. When `generate_smoke_tests: true`
+        is configured, the SDK generates tests in `tests/smoke/` that can be run against a production
+        environment. Features include:
+        - Three validation modes: status-only, structural (types/structure), and strict (exact values)
+        - Per-endpoint validation filtering via include/exclude patterns (e.g., `ServiceName.methodName`)
+        - Environment variable configuration via `SMOKE_TEST_BASE_URL` and `SMOKE_TEST_<PARAM>`
+      type: feat
+  createdAt: "2025-01-07"
+  irVersion: 62
+
 - version: 4.46.7-rc2
   changelogEntry:
     - summary: Fixed Python SDK generation to use native Pydantic field aliases and improved core parsing so wire-key and nested/union validation works correctly across Pydantic v1/v2.

generators/python/src/fern_python/generators/sdk/custom_config.py

@@ -43,6 +43,61 @@ class CustomReadmeSection(pydantic.BaseModel):
     content: str
 
 
+class ValidationFilterConfig(pydantic.BaseModel):
+    """Configuration for filtering which examples get a specific validation type."""
+
+    # List of example patterns to include for this validation type.
+    # If None, all examples are included. If empty list, none are included.
+    # Format: "ServiceName.methodName" or "ServiceName.methodName:exampleName"
+    include_examples: Optional[List[str]] = None
+
+    # List of example patterns to exclude from this validation type.
+    # Format: "ServiceName.methodName" or "ServiceName.methodName:exampleName"
+    exclude_examples: Optional[List[str]] = None
+
+    class Config:
+        extra = pydantic.Extra.forbid
+
+
+class SmokeTestConfig(pydantic.BaseModel):
+    """Configuration for smoke test generation.
+
+    Smoke tests support three layers of validation:
+    1. Status validation: Always runs - verifies the request succeeds (no exception)
+    2. Structural validation: Validates response types/structure match the example
+    3. Strict validation: Validates exact response values match the example
+    """
+
+    # Whether to generate smoke tests. When True, generates tests in tests/smoke/
+    # that can be run against a production environment.
+    generate_smoke_tests: bool = False
+
+    # List of example names to include in smoke test generation.
+    # If None, all user-specified examples are included.
+    # Format: "ServiceName.methodName" or "ServiceName.methodName:exampleName"
+    include_examples: Optional[List[str]] = None
+
+    # List of example names to exclude from smoke test generation.
+    # Format: "ServiceName.methodName" or "ServiceName.methodName:exampleName"
+    exclude_examples: Optional[List[str]] = None
+
+    # Configuration for structural validation (validates types/structure but not exact values).
+    # By default, structural validation runs on all generated tests.
+    structural_validation: ValidationFilterConfig = pydantic.Field(
+        default_factory=lambda: ValidationFilterConfig(include_examples=None, exclude_examples=[])
+    )
+
+    # Configuration for strict validation (validates exact response values).
+    # By default, strict validation is disabled (empty include list).
+    # When strict validation runs, it implies structural validation.
+    strict_validation: ValidationFilterConfig = pydantic.Field(
+        default_factory=lambda: ValidationFilterConfig(include_examples=[], exclude_examples=[])
+    )
+
+    class Config:
+        extra = pydantic.Extra.forbid
+
+
 class SDKCustomConfig(pydantic.BaseModel):
     extra_dependencies: Dict[str, Union[str, DependencyCustomConfig]] = {}
     extra_dev_dependencies: Dict[str, Union[str, BaseDependencyCustomConfig]] = {}
@@ -127,6 +182,10 @@ class SDKCustomConfig(pydantic.BaseModel):
 
     custom_pager_name: Optional[str] = None
 
+    # Configuration for smoke test generation.
+    # Set smoke_test_config.generate_smoke_tests = True to enable.
+    smoke_test_config: SmokeTestConfig = SmokeTestConfig()
+
     class Config:
         extra = pydantic.Extra.forbid
 

generators/python/src/fern_python/generators/sdk/sdk_generator.py

@@ -39,6 +39,7 @@
     ClientWrapperGenerator,
 )
 from fern_python.snippet import SnippetRegistry, SnippetWriter
+from fern_python.snippet.smoke_test_factory import SmokeTestFactory
 from fern_python.snippet.snippet_test_factory import SnippetTestFactory
 from fern_python.utils import build_snippet_writer
 
@@ -362,6 +363,18 @@ def run(
                 ir=ir,
             )
 
+        # Generate smoke tests if enabled
+        if context.custom_config.smoke_test_config.generate_smoke_tests:
+            self._write_smoke_tests(
+                context=context,
+                project=project,
+                generator_exec_wrapper=generator_exec_wrapper,
+                generated_root_client=generated_root_client,
+                base_environment=base_environment,
+                snippet_writer=snippet_writer,
+                ir=ir,
+            )
+
     def postrun(self, *, generator_exec_wrapper: GeneratorExecWrapper) -> None:
         # Finally, run the python-v2 generator.
         pythonv2 = PythonV2Generator(
@@ -660,6 +673,26 @@ def _write_snippet_tests(
     ) -> None:
         snippet_test_factory.tests(ir, snippet_writer)
 
+    def _write_smoke_tests(
+        self,
+        context: SdkGeneratorContext,
+        project: Project,
+        generator_exec_wrapper: GeneratorExecWrapper,
+        generated_root_client: GeneratedRootClient,
+        base_environment: Optional[Union[SingleBaseUrlEnvironmentGenerator, MultipleBaseUrlsEnvironmentGenerator]],
+        snippet_writer: SnippetWriter,
+        ir: ir_types.IntermediateRepresentation,
+    ) -> None:
+        smoke_test_factory = SmokeTestFactory(
+            project=project,
+            context=context,
+            generator_exec_wrapper=generator_exec_wrapper,
+            generated_root_client=generated_root_client,
+            generated_environment=base_environment,
+            smoke_test_config=context.custom_config.smoke_test_config,
+        )
+        smoke_test_factory.generate(ir, snippet_writer)
+
     def get_sorted_modules(self) -> Sequence[str]:
         # always import types/errors before resources (nested packages)
         # to avoid issues with circular imports