fix: set origin when merging chunks (#109)

Signed-off-by: Panos Vagenas <[email protected]>
DS4SD · Dec 13, 2024 · b546c0a · b546c0a
1 parent 2591c70
commit b546c0a
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 6 deletions.
diff --git a/docling_core/transforms/chunker/hybrid_chunker.py b/docling_core/transforms/chunker/hybrid_chunker.py
@@ -98,6 +98,7 @@ def _make_chunk_from_doc_items(
             doc_items=doc_chunk.meta.doc_items[window_start : window_end + 1],
             headings=doc_chunk.meta.headings,
             captions=doc_chunk.meta.captions,
+            origin=doc_chunk.meta.origin,
         )
         new_chunk = DocChunk(text=window_text, meta=meta)
         return new_chunk
@@ -244,6 +245,7 @@ def _merge_chunks_with_matching_metadata(self, chunks: list[DocChunk]):
                         doc_items=window_items,
                         headings=current_headings_and_captions[0],
                         captions=current_headings_and_captions[1],
+                        origin=chunk.meta.origin,
                     )
                     new_chunk = DocChunk(
                         text=window_text,

diff --git a/test/data/chunker/2a_out_chunks.json b/test/data/chunker/2a_out_chunks.json
@@ -95,7 +95,12 @@
                 ],
                 "headings": [
                     "IBM"
-                ]
+                ],
+                "origin": {
+                    "mimetype": "application/pdf",
+                    "binary_hash": 15535403176419637685,
+                    "filename": "wiki.pdf"
+                }
             }
         },
         {
@@ -1000,7 +1005,12 @@
                     "IBM",
                     "Corporate affairs",
                     "Business trends"
-                ]
+                ],
+                "origin": {
+                    "mimetype": "application/pdf",
+                    "binary_hash": 15535403176419637685,
+                    "filename": "wiki.pdf"
+                }
             }
         },
         {
@@ -1062,8 +1072,13 @@
                     "IBM",
                     "Corporate affairs",
                     "Business trends"
-                ]
+                ],
+                "origin": {
+                    "mimetype": "application/pdf",
+                    "binary_hash": 15535403176419637685,
+                    "filename": "wiki.pdf"
+                }
             }
         }
     ]
-}
+}
diff --git a/test/data/chunker/2c_out_chunks.json b/test/data/chunker/2c_out_chunks.json
@@ -191,7 +191,12 @@
                 ],
                 "headings": [
                     "IBM"
-                ]
+                ],
+                "origin": {
+                    "mimetype": "application/pdf",
+                    "binary_hash": 15535403176419637685,
+                    "filename": "wiki.pdf"
+                }
             }
         },
         {
@@ -448,7 +453,12 @@
                     "IBM",
                     "Corporate affairs",
                     "Business trends"
-                ]
+                ],
+                "origin": {
+                    "mimetype": "application/pdf",
+                    "binary_hash": 15535403176419637685,
+                    "filename": "wiki.pdf"
+                }
             }
         }
     ]