Merge pull request #14 from AstraBert/v1.0.1-branch

v1.0.1 bugfixes

Author: AstraBert

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sentrev"
-version = "1.0.0"
+version = "1.0.1"
 authors = [
     { name="Clelia (Astra) Bertelli", email="[email protected]" },
 ]

src/sentrev/evaluator.py

@@ -383,13 +383,15 @@ def evaluate_dense_retrieval(
             client.delete_collection(collection_name)
     performances_df = pd.DataFrame.from_dict(performances)
     performances_df.to_csv(csv_path, index=False)
+    csv_name = os.path.basename(csv_path)
+    csv_path_base = os.path.dirname(csv_path)
     if plot:
-        path_time = csv_path.split(".")[0] + "_times.png"
-        path_sr = csv_path.split(".")[0] + "_success_rate.png"
-        path_mrr = csv_path.split(".")[0] + "_mrr.png"
-        path_co2 = csv_path.split(".")[0] + "_co2.png"
-        path_precision = csv_path.split(".")[0] + "_precision.png"
-        path_nonrelevant = csv_path.split(".")[0] + "_nonrelevant.png"
+        path_time = csv_path_base + "/" + csv_name.split(".")[0] + "_times.png"
+        path_sr = csv_path_base + "/" + csv_name.split(".")[0] + "_success_rate.png"
+        path_mrr = csv_path_base + "/" + csv_name.split(".")[0] + "_mrr.png"
+        path_co2 = csv_path_base + "/" + csv_name.split(".")[0] + "_co2.png"
+        path_precision = csv_path_base + "/" + csv_name.split(".")[0] + "_precision.png"
+        path_nonrelevant = csv_path_base + "/" + csv_name.split(".")[0] + "_nonrelevant.png"
         X = performances["encoder"]
         y_times = performances["average_time"]
         yerr_times = performances["stdev_time"]
@@ -747,13 +749,15 @@ def evaluate_sparse_retrieval(
             client.delete_collection(collection_name)
     performances_df = pd.DataFrame.from_dict(performances)
     performances_df.to_csv(csv_path, index=False)
+    csv_name = os.path.basename(csv_path)
+    csv_path_base = os.path.dirname(csv_path)
     if plot:
-        path_time = csv_path.split(".")[0] + "_times.png"
-        path_sr = csv_path.split(".")[0] + "_success_rate.png"
-        path_mrr = csv_path.split(".")[0] + "_mrr.png"
-        path_co2 = csv_path.split(".")[0] + "_co2.png"
-        path_precision = csv_path.split(".")[0] + "_precision.png"
-        path_nonrelevant = csv_path.split(".")[0] + "_nonrelevant.png"
+        path_time = csv_path_base + "/" + csv_name.split(".")[0] + "_times.png"
+        path_sr = csv_path_base + "/" + csv_name.split(".")[0] + "_success_rate.png"
+        path_mrr = csv_path_base + "/" + csv_name.split(".")[0] + "_mrr.png"
+        path_co2 = csv_path_base + "/" + csv_name.split(".")[0] + "_co2.png"
+        path_precision = csv_path_base + "/" + csv_name.split(".")[0] + "_precision.png"
+        path_nonrelevant = csv_path_base + "/" + csv_name.split(".")[0] + "_nonrelevant.png"
         X = performances["encoder"]
         y_times = performances["average_time"]
         yerr_times = performances["stdev_time"]

src/sentrev/utils.py

@@ -195,14 +195,14 @@ def collect_data(self):
         for text in self.pages:
             contents = text.page_content.split("\n")
             contents = remove_items(contents, "")
-            for content in contents:
-                self.documents.append(
-                    {
-                        "text": content,
-                        "source": text.metadata["source"],
-                        "page": str(text.metadata["page"]),
-                    }
-                )
+            content = "\n".join(contents)
+            self.documents.append(
+                {
+                    "text": content,
+                    "source": text.metadata["source"],
+                    "page": str(text.metadata["page"]),
+                }
+            )
         return self.documents
 
     def qdrant_collection_and_upload(self):