Update to pydantic v2

.github/workflows/ci_tests.yml

@@ -20,7 +20,6 @@ jobs:
 
       # pylint & safety
       python_version_pylint_safety: '3.9'
-
       run_pylint: false
       run_safety: true
 
@@ -32,12 +31,3 @@ jobs:
 
       # Documentation
       run_build_docs: false
-
-  ruff:
-    name: Ruff
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - uses: chartboost/ruff-action@v1

.pre-commit-config.yaml

@@ -4,7 +4,7 @@
 #   pre-commit autoupdate
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: end-of-file-fixer
     - id: debug-statements
@@ -16,15 +16,15 @@ repos:
       args: [--markdown-linebreak-ext=md]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.290
+    rev: v0.0.292
     hooks:
       - id: ruff
         args:
         - "--fix"
         - "--exit-non-zero-on-fix"
 
   - repo: https://github.com/ambv/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
     - id: black
 
@@ -36,9 +36,9 @@ repos:
       files: ^s7/.*$
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.6.0
     hooks:
     - id: mypy
       additional_dependencies:
-      - pydantic
+      - pydantic>=2,<3
       - types-PyYAML

pyproject.toml

@@ -29,17 +29,17 @@ dynamic = ["version"]
 dependencies = [
     "graphviz",
     "Jinja2",
-    "oteapi-core",
-    "otelib",
-    "pydantic",
+    "oteapi-core>=0.6.0,<1",
+    "otelib>=0.4.0,<1",
+    "pydantic~=2.4",
     "pymongo",
     "pyyaml",
     "rdflib",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "pre-commit ~=2.18",
+    "pre-commit~=2.18",
 ]
 
 [project.urls]
@@ -62,11 +62,3 @@ warn_unused_configs = true
 show_error_codes = true
 allow_redefinition = true
 plugins = ["pydantic.mypy"]
-
-# [tool.pytest.ini_options]
-# minversion = "7.4"
-# addopts = "-rs --cov=s7 --cov-report=term --durations=10"
-# filterwarnings = [
-#     "ignore:.*imp module.*:DeprecationWarning",
-#     "ignore:.*_yaml extension module.*:DeprecationWarning"
-# ]

s7/factories/datasource_factory.py

@@ -12,24 +12,39 @@
 """
 import json
 from pathlib import Path
-from typing import Any, Optional, Type, Union
+from typing import TYPE_CHECKING, Annotated
 
 import yaml
 from oteapi.models import ResourceConfig
 from otelib import OTEClient
 from pydantic import Field, create_model
 
 from s7.pydantic_models.oteapi import HashableResourceConfig
-from s7.pydantic_models.soft7 import SOFT7DataEntity, SOFT7Entity
+from s7.pydantic_models.soft7 import SOFT7DataEntity, SOFT7Entity, map_soft_to_py_types
 
+if TYPE_CHECKING:  # pragma: no cover
+    from typing import Any, Optional, Union, Protocol
 
-def _get_property(config: HashableResourceConfig, url: Optional[str] = None) -> Any:
+    class GetProperty(Protocol):
+        """Protocol for getting a property."""
+
+        def __call__(self, name: str) -> Any:
+            ...
+
+
+def _get_property(
+    config: HashableResourceConfig, url: "Optional[str]" = None
+) -> "GetProperty":
     """Get a property."""
     client = OTEClient(url or "http://localhost:8080")
-    data_resource = client.create_dataresource(**config.dict())
-    result: dict[str, Any] = json.loads(data_resource.get())
+    data_resource = client.create_dataresource(**config.model_dump())
+    result: "dict[str, Any]" = json.loads(data_resource.get())
+
+    # Remove unused variables from memory
+    del client
+    del data_resource
 
-    def __get_property(name: str) -> Any:
+    def __get_property(name: str) -> "Any":
         if name in result:
             return result[name]
 
@@ -38,27 +53,10 @@ def __get_property(name: str) -> Any:
     return __get_property
 
 
-# def _get_property_local(
-#     config: HashableResourceConfig,
-# ) -> Any:
-#     """TEMPORARY - Get a property - local."""
-#     from s7.temporary.xlsparser import XLSParser
-
-#     parser = XLSParser(config.configuration).get()
-
-#     def __get_property_local(name: str) -> Any:
-#         if name in parser:
-#             return parser[name]
-
-#         raise ValueError(f"Could find no data for {name!r}")
-
-#     return __get_property_local
-
-
 def create_entity(
-    data_model: Union[SOFT7Entity, Path, str, dict[str, Any]],
-    resource_config: Union[HashableResourceConfig, ResourceConfig, dict[str, Any]],
-) -> Type[SOFT7DataEntity]:
+    data_model: "Union[SOFT7Entity, dict[str, Any], Path, str]",
+    resource_config: "Union[HashableResourceConfig, ResourceConfig, dict[str, Any]]",
+) -> type[SOFT7DataEntity]:
     """Create and return a SOFT7 entity wrapped as a pydantic model.
 
     Parameters:
@@ -73,56 +71,80 @@ def create_entity(
         A SOFT7 entity class wrapped as a pydantic data model.
 
     """
+    # Handle the case of data model being a string/path to a YAML file
     if isinstance(data_model, (str, Path)):
-        if not Path(data_model).resolve().exists():
+        data_model_path = Path(data_model).resolve()
+
+        if not data_model_path.exists():
             raise FileNotFoundError(
-                f"Could not find a data model YAML file at {data_model!r}"
+                f"Could not find a data model YAML file at {data_model_path}"
             )
-        data_model: dict[str, Any] = yaml.safe_load(  # type: ignore[no-redef]
-            Path(data_model).resolve().read_text(encoding="utf8")
-        )
+
+        data_model = yaml.safe_load(data_model_path.read_text(encoding="utf8"))
+
+        if not isinstance(data_model, dict):
+            raise TypeError(
+                f"Data model YAML file at {data_model_path} did not contain a "
+                "dictionary"
+            )
+
+    # Now the data model is either a SOFT7Entity instance or a dictionary, ready to be
+    # used to create the SOFT7Entity instance.
     if isinstance(data_model, dict):
         data_model = SOFT7Entity(**data_model)
+
     if not isinstance(data_model, SOFT7Entity):
-        raise TypeError("data_model must be a 'SOFT7Entity'")
+        raise TypeError(
+            f"data_model must be a 'SOFT7Entity', instead it was a {type(data_model)}"
+        )
 
+    # Handle the case of resource_config being a dictionary or a "pure" OTEAPI Core
+    # ResourceConfig. We need to convert it to a HashableResourceConfig.
     if isinstance(resource_config, dict):
         resource_config = HashableResourceConfig(**resource_config)
+
     if isinstance(resource_config, ResourceConfig) and not isinstance(
         resource_config, HashableResourceConfig
     ):
         resource_config = HashableResourceConfig(
-            **resource_config.dict(exclude_defaults=True)
+            **resource_config.model_dump(exclude_defaults=True)
         )
-    if not isinstance(resource_config, HashableResourceConfig):
-        raise TypeError("resource_config must be a 'ResourceConfig' (from oteapi-core)")
-    resource_config: HashableResourceConfig  # type: ignore[no-redef]  # Satisfy mypy
 
-    if any(property_name.startswith("_") for property_name in data_model.properties):
-        raise ValueError(
-            "data model property names may not start with an underscore (_)"
+    if not isinstance(resource_config, HashableResourceConfig):
+        raise TypeError(
+            "resource_config must be a 'HashableResourceConfig', instead it was a "
+            f"{type(resource_config)}"
         )
 
-    return create_model(  # type: ignore[call-overload]
-        __model_name="DataSourceEntity",
+    field_definitions = {
+        property_name: Annotated[
+            map_soft_to_py_types[property_value.type_],
+            Field(
+                # mypy cannot recognize that resource_config is HashableResourceConfig
+                # even though it's checked above (see isinstance() check above)
+                default_factory=lambda: _get_property(resource_config),  # type: ignore[arg-type]  # noqa: E501
+                description=property_value.description or "",
+                title=property_name.replace(" ", "_"),
+                validate_default=True,
+                json_schema_extra={
+                    f"x-{field}": getattr(property_value, field)
+                    for field in property_value.model_fields
+                    if (
+                        field not in ("description", "type_", "shape")
+                        and getattr(property_value, field)
+                    )
+                },
+            ),
+        ]
+        for property_name, property_value in data_model.properties.items()
+    }
+
+    return create_model(
+        "DataSourceEntity",
         __config__=None,
         __base__=SOFT7DataEntity,
         __module__=__name__,
         __validators__=None,
         __cls_kwargs__=None,
-        **{
-            property_name: Field(  # type: ignore[pydantic-field]
-                default_factory=lambda: _get_property(resource_config),
-                description=property_value.description or "",
-                title=property_name.replace(" ", "_"),
-                type=property_value.type_.py_cls,
-                **{
-                    f"x-{field}": getattr(property_value, field)
-                    for field in property_value.__fields__
-                    if field not in ("description", "type_", "shape")
-                    and getattr(property_value, field)
-                },
-            )
-            for property_name, property_value in data_model.properties.items()
-        },
+        **field_definitions,
     )