fix: change order of JSON Schema to search mapper transformations (#32)

* fix: change order of JSON Schema to search mapper transformations In the JSON Schema to search mapper, the suppression flags need to be addressed first, otherwise the jsonref.replace_refs function may remove them. Signed-off-by: Cesar Berrospi Ramis <[email protected]> * build: update dependencies Updating pydantic from 2.8.2 to 2.9.2 triggers a change in JSON Schema from models: the lists with only 1 element get flatten. Signed-off-by: Cesar Berrospi Ramis <[email protected]> * chore: improve verbose in JSON Schema to search mapper test Signed-off-by: Cesar Berrospi Ramis <[email protected]> --------- Signed-off-by: Cesar Berrospi Ramis <[email protected]>
DS4SD · Sep 26, 2024 · a4ddd14 · a4ddd14
1 parent b49e93e
commit a4ddd14
Show file tree

Hide file tree

Showing 7 changed files with 597 additions and 498 deletions.
diff --git a/docling_core/search/json_schema_to_search_mapper.py b/docling_core/search/json_schema_to_search_mapper.py
@@ -8,7 +8,7 @@
 from copy import deepcopy
 from typing import Any, Optional, Pattern, Tuple, TypedDict
 
-from jsonref import JsonRef
+from jsonref import replace_refs
 
 
 class SearchIndexDefinition(TypedDict):
@@ -95,7 +95,11 @@ def get_index_definition(self, schema: dict) -> SearchIndexDefinition:
         which define the fields, their data types, and other specifications to index
         JSON documents into a Lucene index.
         """
-        mapping = JsonRef.replace_refs(schema)
+        mapping = deepcopy(schema)
+
+        mapping = self._suppress(mapping, self._suppress_key)
+
+        mapping = replace_refs(mapping)
 
         mapping = self._merge_unions(mapping)
 
@@ -105,8 +109,6 @@ def get_index_definition(self, schema: dict) -> SearchIndexDefinition:
 
         mapping = self._remove_keys(mapping, self._rm_keys)
 
-        mapping = self._suppress(mapping, self._suppress_key)
-
         mapping = self._translate_keys_re(mapping)
 
         mapping = self._clean(mapping)

diff --git a/docs/Document.json b/docs/Document.json
@@ -323,11 +323,7 @@
           "type": "string"
         },
         "bounding_box": {
-          "allOf": [
-            {
-              "$ref": "#/$defs/BoundingBoxContainer"
-            }
-          ],
+          "$ref": "#/$defs/BoundingBoxContainer",
           "x-es-suppress": true
         },
         "prov": {

diff --git a/docs/Document.md b/docs/Document.md
@@ -6052,7 +6052,7 @@ Must be one of:
 | **Type**                  | `object`                                                                  |
 | **Required**              | Yes                                                                       |
 | **Additional properties** | [[Any type: allowed]](# "Additional Properties of any type are allowed.") |
-| **Defined in**            |                                                                           |
+| **Defined in**            | #/$defs/BoundingBoxContainer                                              |
 
 **Description:** Bounding box container.
 

diff --git a/docs/Generic.json b/docs/Generic.json
@@ -58,11 +58,7 @@
       "x-es-type": "text"
     },
     "file-info": {
-      "allOf": [
-        {
-          "$ref": "#/$defs/FileInfoObject"
-        }
-      ],
+      "$ref": "#/$defs/FileInfoObject",
       "description": "Minimal identification information of the document within a collection.",
       "title": "Document information"
     }

diff --git a/docs/Generic.md b/docs/Generic.md
@@ -75,7 +75,7 @@
 | **Type**                  | `object`                                                                  |
 | **Required**              | Yes                                                                       |
 | **Additional properties** | [[Any type: allowed]](# "Additional Properties of any type are allowed.") |
-| **Defined in**            |                                                                           |
+| **Defined in**            | #/$defs/FileInfoObject                                                    |
 
 **Description:** Minimal identification information of the document within a collection.
 

diff --git a/poetry.lock b/poetry.lock
diff --git a/test/test_json_schema_to_search_mapper.py b/test/test_json_schema_to_search_mapper.py
@@ -51,9 +51,10 @@ def test_json_schema_to_search_mapper_0():
     index_ref = _load(filename)
 
     diff = jsondiff.diff(index_ref, index_def)
-    print(json.dumps(index_def, indent=2))
-    print(diff)
-    assert index_def == index_ref
+    # print(json.dumps(index_def, indent=2))
+    assert (
+        index_def == index_ref
+    ), f"Error in search mappings of ExportedCCSDocument. Difference:\n{json.dumps(diff, indent=2)}"
 
 
 def test_json_schema_to_search_mapper_1():
@@ -99,6 +100,7 @@ def test_json_schema_to_search_mapper_1():
     index_ref = _load(filename)
 
     diff = jsondiff.diff(index_ref, index_def)
-    # print(json.dumps(index_def,indent=2))
-    print(diff)
-    assert index_def == index_ref
+    # print(json.dumps(index_def, indent=2))
+    assert (
+        index_def == index_ref
+    ), f"Error in search mappings of Record. Difference:\n{json.dumps(diff, indent=2)}"