Add capability to support Collection and Default experiment during parsing (#223)

* Add configurable collection type for parser with COLLECTION as default (#222)

* Initial plan

* Add collection_type parameter to parser infrastructure

- Add collection_type parameter to run_parser() with default value "COLLECTION"
- Add --collection-type CLI option with choices between COLLECTION and DEFAULT_EXPERIMENT
- Update tests to cover both collection types and default behavior
- Update documentation to explain the new parameter

---------

Co-authored-by: copilot-swe-agent[bot] <[email protected]>
Co-authored-by: JosePizarro3 <[email protected]>

* Minor check in run_parser for collection_type

Fixes on documentation

Author: JosePizarro3

bam_masterdata/cli/cli.py

@@ -617,7 +617,14 @@ def push_to_openbis(file_path, datamodel_path):
     required=False,
     help="OpenBIS space name",
 )
-def parser(files_parser, project_name, collection_name, space_name):
+@click.option(
+    "--collection-type",
+    "collection_type",  # alias
+    type=click.Choice(["COLLECTION", "DEFAULT_EXPERIMENT"], case_sensitive=False),
+    default="COLLECTION",
+    help="Type of collection to create in openBIS. Options are 'COLLECTION' or 'DEFAULT_EXPERIMENT'. Defaults to 'COLLECTION'.",
+)
+def parser(files_parser, project_name, collection_name, space_name, collection_type):
     parser_map = {}  # TODO load from configuration from yaml file
     parse_file_dict = {}
     for parser_key, filepath in files_parser:
@@ -636,6 +643,7 @@ def parser(files_parser, project_name, collection_name, space_name):
         project_name=project_name,
         collection_name=collection_name,
         files_parser=parse_file_dict,
+        collection_type=collection_type.upper(),
     )
 
 

bam_masterdata/cli/run_parser.py

@@ -15,6 +15,7 @@ def run_parser(
     project_name: str = "PROJECT",
     collection_name: str = "",
     files_parser: dict[AbstractParser, list[str]] = {},
+    collection_type: str = "COLLECTION",
 ) -> None:
     """
     Run the parsers on the specified files and collect the results.
@@ -26,6 +27,7 @@ def run_parser(
         project_name (str): The project in openBIS where the entities will be stored.
         collection_name (str): The collection in openBIS where the entities will be stored.
         files_parser (dict): A dictionary where keys are parser instances and values are lists of file paths to be parsed. E.g., {MasterdataParserExample(): ["path/to/file.json", "path/to/another_file.json"]}
+        collection_type (str): The type of collection to create in openBIS. Options are "COLLECTION" or "DEFAULT_EXPERIMENT". Defaults to "COLLECTION".
     """
     # Ensure openbis is provided
     if openbis is None:
@@ -41,6 +43,12 @@ def run_parser(
             "No files or parsers to parse. Please provide valid file paths or contact an Admin to add missing parser."
         )
         return
+    # Ensure collection_type is valid
+    if collection_type not in ["COLLECTION", "DEFAULT_EXPERIMENT"]:
+        logger.error(
+            f"Invalid collection_type '{collection_type}'. Must be either 'COLLECTION' or 'DEFAULT_EXPERIMENT'."
+        )
+        return
 
     # Specify the space
     try:
@@ -91,7 +99,7 @@ def run_parser(
             logger.info("Replacing collection code with uppercase and underscores.")
             collection_openbis = openbis.new_collection(
                 code=collection_name.replace(" ", "_").upper(),
-                type="DEFAULT_EXPERIMENT",
+                type=collection_type,
                 project=project,
             )
         collection_openbis.save()

docs/tutorials/parsing.md

@@ -54,6 +54,7 @@ The whole process then depends on the inputs and output `run_parser()`:
     - `space_name`: _string_, this coincides with the space name you have associated in your openBIS instance
     - `project_name`: _string_, the name of the project where you want to store the parsed objects
     - `collection_name`: _string_, optional, the name of the collection you want to store the parsed objects
+    - `collection_type`: _string_, optional, the type of collection to create in openBIS. Options are `"COLLECTION"` or `"DEFAULT_EXPERIMENT"`. Defaults to `"COLLECTION"`.
     - `files_parser`: _dict_, a dictionary where the keys are the parser class instances (see below) and the values are the files you want to pass to the specific parser class
 - **Output**: the metadata objects and relationships mapped to your space/project/collection in openBIS
 
@@ -193,10 +194,18 @@ run_parser(
     space_name="MY_RESEARCH_SPACE",
     project_name="MY_PROJECT",
     collection_name="MY_EXPERIMENT_COLLECTION",
-    files_parser=files_parser
+    files_parser=files_parser,
+    collection_type="COLLECTION"  # Optional: defaults to "COLLECTION"
 )
 ```
 
+!!! tip "Collection Types"
+    You can choose between two collection types:
+    - `"COLLECTION"` (default): A general-purpose collection type
+    - `"DEFAULT_EXPERIMENT"`: A collection type designed for experiments with additional metadata fields like start/end dates, experimental goals, etc.
+
+    If you don't specify `collection_type`, it defaults to `"COLLECTION"`. We recommend keeping the default `collection_type`, as future openBIS releases will get rid of the collection concept and we will, as a result, deprecate this feature.
+
 ### What Happens Next?
 
 When you call `run_parser()`, the following steps occur automatically:
@@ -235,7 +244,7 @@ run_parser(
     space_name="MY_RESEARCH_SPACE",
     project_name="MY_PROJECT",
     collection_name="MULTI_DATA_COLLECTION",
-    files_parser=files_parser
+    files_parser=files_parser,
 )
 ```
 
@@ -456,7 +465,7 @@ def main():
         space_name="RESEARCH_SPACE",
         project_name="AUTOMATION_TEST",
         collection_name="BATCH_EXPERIMENTS",
-        files_parser=files_parser
+        files_parser=files_parser,
     )
 
     print("Metadata injection complete!")

tests/cli/test_run_parser.py

@@ -67,6 +67,7 @@ def test_run_parser_with_test_parser(cleared_log_storage, mock_openbis):
         project_name="TEST_PROJECT",
         collection_name="TEST_COLLECTION",
         files_parser=files_parser,
+        collection_type="COLLECTION",
     )
 
     # Check that objects were created in openbis
@@ -105,6 +106,7 @@ def get_object_mock(path):
         project_name="TEST_PROJECT",
         collection_name="TEST_COLLECTION",
         files_parser=files_parser,
+        collection_type="COLLECTION",
     )
 
     # Only 2 instruments are created (the person is referenced but not persisted)
@@ -122,4 +124,44 @@ def get_object_mock(path):
     )
 
 
+def test_run_parser_with_default_experiment_type(cleared_log_storage, mock_openbis):
+    """Test run_parser with DEFAULT_EXPERIMENT collection type."""
+    file = "./tests/data/cli/test_parser.txt"
+    files_parser = {TestParser(): [file]}
+    run_parser(
+        openbis=mock_openbis,
+        space_name="USERNAME_SPACE",
+        project_name="TEST_PROJECT",
+        collection_name="TEST_COLLECTION",
+        files_parser=files_parser,
+        collection_type="DEFAULT_EXPERIMENT",
+    )
+
+    # Check that objects were created in openbis
+    assert len(mock_openbis._objects) == 1
+
+    # Check logs for success messages
+    assert any("Added test object" in log["event"] for log in cleared_log_storage)
+
+
+def test_run_parser_defaults_to_collection_type(cleared_log_storage, mock_openbis):
+    """Test that run_parser defaults to COLLECTION type when not specified."""
+    file = "./tests/data/cli/test_parser.txt"
+    files_parser = {TestParser(): [file]}
+    # Call without specifying collection_type - should default to "COLLECTION"
+    run_parser(
+        openbis=mock_openbis,
+        space_name="USERNAME_SPACE",
+        project_name="TEST_PROJECT",
+        collection_name="TEST_COLLECTION",
+        files_parser=files_parser,
+    )
+
+    # Check that objects were created in openbis
+    assert len(mock_openbis._objects) == 1
+
+    # Check logs for success messages
+    assert any("Added test object" in log["event"] for log in cleared_log_storage)
+
+
 # TODO add other tests for the different situations in `run_parser()` and parsers from `conftest.py`