add dedupe filters

Author: SuaYoo

backend/btrixcloud/basecrawls.py

@@ -674,6 +674,7 @@ async def list_all_base_crawls(
         tags: list[str] | None = None,
         tag_match: ListFilterType | None = None,
         collection_id: Optional[UUID] = None,
+        dedupe_coll_id: Optional[UUID] = None,
         states: Optional[List[str]] = None,
         first_seed: Optional[str] = None,
         type_: Optional[str] = None,
@@ -790,6 +791,9 @@ async def list_all_base_crawls(
         if description:
             aggregate.extend([{"$match": {"description": description}}])
 
+        if dedupe_coll_id:
+            aggregate.extend([{"$match": {"dedupeCollId": dedupe_coll_id}}])
+
         if collection_id:
             aggregate.extend([{"$match": {"collectionIds": {"$in": [collection_id]}}}])
 
@@ -1078,6 +1082,7 @@ async def list_all_base_crawls(
             ),
         ] = ListFilterType.AND,
         collectionId: Optional[UUID] = None,
+        dedupeCollId: Optional[UUID] = None,
         crawlType: Optional[str] = None,
         cid: Optional[UUID] = None,
         sortBy: Optional[str] = "finished",
@@ -1110,6 +1115,7 @@ async def list_all_base_crawls(
             tags=tags,
             tag_match=tag_match,
             collection_id=collectionId,
+            dedupe_coll_id=dedupeCollId,
             states=states,
             first_seed=firstSeed,
             type_=crawlType,

backend/btrixcloud/colls.py

@@ -539,6 +539,7 @@ async def list_collections(
         sort_direction: int = 1,
         name: Optional[str] = None,
         name_prefix: Optional[str] = None,
+        has_dedupe_index: Optional[bool] = None,
         access: Optional[str] = None,
         headers: Optional[dict] = None,
     ):
@@ -556,6 +557,9 @@ async def list_collections(
             regex_pattern = f"^{name_prefix}"
             match_query["name"] = {"$regex": regex_pattern, "$options": "i"}
 
+        if has_dedupe_index is not None:
+            match_query["hasDedupeIndex"] = has_dedupe_index
+
         if public_colls_out:
             match_query["access"] = CollAccessType.PUBLIC
         elif access:
@@ -1123,6 +1127,7 @@ async def list_collection_all(
         sortDirection: int = 1,
         name: Optional[str] = None,
         namePrefix: Optional[str] = None,
+        hasDedupeIndex: Optional[bool] = None,
         access: Optional[str] = None,
     ):
         # pylint: disable=duplicate-code
@@ -1134,6 +1139,7 @@ async def list_collection_all(
             sort_direction=sortDirection,
             name=name,
             name_prefix=namePrefix,
+            has_dedupe_index=hasDedupeIndex,
             access=access,
             headers=dict(request.headers),
         )

backend/btrixcloud/crawlconfigs.py

@@ -814,6 +814,7 @@ async def get_crawl_configs(
         description: Optional[str] = None,
         tags: Optional[List[str]] = None,
         tag_match: Optional[ListFilterType] = ListFilterType.AND,
+        dedupe_coll_id: Optional[UUID] = None,
         last_crawl_state: list[TYPE_ALL_CRAWL_STATES] | None = None,
         schedule: Optional[bool] = None,
         is_crawl_running: Optional[bool] = None,
@@ -868,6 +869,9 @@ async def get_crawl_configs(
         if first_seed:
             aggregate.extend([{"$match": {"firstSeed": first_seed}}])
 
+        if dedupe_coll_id:
+            aggregate.extend([{"$match": {"dedupeCollId": first_seed}}])
+
         if sort_by:
             if sort_by not in ALLOWED_SORT_KEYS:
                 raise HTTPException(status_code=400, detail="invalid_sort_by")
@@ -1728,6 +1732,7 @@ async def get_crawl_configs(
                 description='Defaults to `"and"` if omitted',
             ),
         ] = ListFilterType.AND,
+        dedupeCollId: Optional[UUID] = None,
         last_crawl_state: Annotated[
             list[TYPE_ALL_CRAWL_STATES] | None,
             Query(alias="lastCrawlState", title="Last Crawl State"),
@@ -1765,6 +1770,7 @@ async def get_crawl_configs(
             description=description,
             tags=tags,
             tag_match=tag_match,
+            dedupe_coll_id=dedupeCollId,
             last_crawl_state=last_crawl_state,
             schedule=schedule,
             is_crawl_running=is_crawl_running,