revert smk params import

studio/app/common/core/rules/runner.py

@@ -54,8 +54,6 @@ def run(cls, __rule: Rule, last_output, run_script_path: str):
 
             cls.__set_func_start_timestamp(os.path.dirname(__rule.output))
 
-            __rule.params.update(__rule.smk_params)
-
             # output_info
             output_info = cls.__execute_function(
                 __rule.path,

studio/app/common/core/snakemake/smk.py

@@ -9,7 +9,6 @@ class Rule:
     input: list
     return_arg: Union[str, Dict[str, str]]
     params: dict
-    smk_params: dict
     output: str
     type: str
     nwbfile: dict = None

studio/app/common/core/snakemake/smk_builder.py

@@ -5,7 +5,6 @@ class RuleBuilder:
     def __init__(self) -> None:
         self._input = None
         self._return_arg = None
-        self._smk_params = None
         self._params = None
         self._output = None
         self._type = None
@@ -28,10 +27,6 @@ def set_return_arg(self, return_arg) -> "RuleBuilder":
         self._return_arg = return_arg
         return self
 
-    def set_smk_params(self, smk_params) -> "RuleBuilder":
-        self._smk_params = smk_params
-        return self
-
     def set_params(self, params) -> "RuleBuilder":
         self._params = params
         return self
@@ -64,7 +59,6 @@ def build(self) -> Rule:
         return Rule(
             input=self._input,
             return_arg=self._return_arg,
-            smk_params=self._smk_params,
             params=self._params,
             output=self._output,
             type=self._type,

studio/app/common/core/snakemake/snakemake_reader.py

@@ -7,7 +7,6 @@ def read(cls, rule):
         return Rule(
             input=rule["input"],
             return_arg=rule["return_arg"],
-            smk_params=rule["smk_params"],
             params=rule["params"],
             output=rule["output"],
             type=rule["type"],

studio/app/common/core/snakemake/snakemake_rule.py

@@ -13,14 +13,12 @@ def __init__(
         self,
         workspace_id: str,
         unique_id: str,
-        smk_params: Dict,
         node: Node,
         edgeDict: Dict[str, Edge],
         nwbfile=None,
     ) -> None:
         self._workspace_id = workspace_id
         self._unique_id = unique_id
-        self._smk_params = smk_params
         self._node = node
         self._edgeDict = edgeDict
         self._nwbfile = nwbfile
@@ -40,7 +38,6 @@ def __init__(
                 self._node.data.path, workspace_id=self._workspace_id
             )
             .set_return_arg(_return_name)
-            .set_smk_params(self._smk_params)
             .set_params(self._node.data.param)
             .set_output(_output_file)
             .set_nwbfile(self._nwbfile)
@@ -100,7 +97,6 @@ def algo(self, nodeDict: Dict[str, Node]) -> Rule:
         return (
             self.builder.set_input(algo_input)
             .set_return_arg(return_arg_names)
-            .set_smk_params(self._smk_params)
             .set_params(params)
             .set_output(algo_output)
             .set_path(self._node.data.path)

studio/app/common/core/workflow/workflow_runner.py

@@ -66,12 +66,12 @@ def create_workflow_unique_id() -> str:
         return new_unique_id
 
     def run_workflow(self, background_tasks):
+        self.set_smk_config()
         snakemake_params: SmkParam = get_typecheck_params(
             self.runItem.snakemakeParam, "snakemake"
         )
         snakemake_params = SmkParamReader.read(snakemake_params)
         snakemake_params.forcerun = self.runItem.forceRunList
-        self.set_smk_config(snakemake_params)
         if len(snakemake_params.forcerun) > 0:
             delete_dependencies(
                 workspace_id=self.workspace_id,
@@ -84,8 +84,8 @@ def run_workflow(self, background_tasks):
             snakemake_execute, self.workspace_id, self.unique_id, snakemake_params
         )
 
-    def set_smk_config(self, snakemake_params: SmkParam):
-        rules, last_output = self.rulefile(snakemake_params)
+    def set_smk_config(self):
+        rules, last_output = self.rulefile()
         flow_config = FlowConfig(
             rules=rules,
             last_output=last_output,
@@ -94,7 +94,7 @@ def set_smk_config(self, snakemake_params: SmkParam):
             self.workspace_id, self.unique_id, asdict(flow_config)
         )
 
-    def rulefile(self, snakemake_params: SmkParam) -> Dict[str, Rule]:
+    def rulefile(self) -> Dict[str, Rule]:
         endNodeList = self.get_endNodeList()
 
         nwbfile = get_typecheck_params(self.runItem.nwbParam, "nwb")
@@ -107,7 +107,6 @@ def rulefile(self, snakemake_params: SmkParam) -> Dict[str, Rule]:
                 data_common_rule = SmkRule(
                     workspace_id=self.workspace_id,
                     unique_id=self.unique_id,
-                    smk_params=snakemake_params,
                     node=node,
                     edgeDict=self.edgeDict,
                     nwbfile=nwbfile,
@@ -134,7 +133,6 @@ def rulefile(self, snakemake_params: SmkParam) -> Dict[str, Rule]:
                 algo_rule = SmkRule(
                     workspace_id=self.workspace_id,
                     unique_id=self.unique_id,
-                    smk_params=snakemake_params,
                     node=node,
                     edgeDict=self.edgeDict,
                 ).algo(nodeDict=self.nodeDict)

studio/app/optinist/wrappers/caiman/cnmf.py

@@ -13,10 +13,7 @@
 from studio.app.common.dataclass import ImageData
 from studio.app.optinist.core.nwb.nwb import NWBDATASET
 from studio.app.optinist.dataclass import EditRoiData, FluoData, IscellData, RoiData
-from studio.app.optinist.wrappers.optinist.utils import (
-    recursive_flatten_params,
-    split_dictionary,
-)
+from studio.app.optinist.wrappers.optinist.utils import recursive_flatten_params
 
 logger = AppLogger.get_logger()
 
@@ -187,22 +184,18 @@ def caiman_cnmf(
     if "dview" in locals():
         stop_server(dview=dview)  # noqa: F821
 
-    n_processes = None
+    # TODO: Add parameters for node
+    n_processes = 1
     dview = None
     # This process launches another process to run the CNMF algorithm,
     # so this node use at least 2 core.
-    # TODO: The minimum number of cores in the snakamake setting should be 2
-    if smk_parms["cores"] == 1:
-        c, dview, n_processes = setup_cluster(
-            backend="single", n_processes=smk_parms["cores"], single_thread=True
-        )
-    elif smk_parms["cores"] == 2:
+    if n_processes == 1:
         c, dview, n_processes = setup_cluster(
-            backend="single", n_processes=smk_parms["cores"] - 1, single_thread=True
+            backend="single", n_processes=n_processes, single_thread=True
         )
     else:
         c, dview, n_processes = setup_cluster(
-            backend="multiprocessing", n_processes=smk_parms["cores"] - 1
+            backend="multiprocessing", n_processes=n_processes
         )
     logger.info(f"n_processes: {n_processes}")
 

studio/app/optinist/wrappers/caiman/cnmf_multisession.py

@@ -13,10 +13,7 @@
     util_download_model_files,
     util_get_memmap,
 )
-from studio.app.optinist.wrappers.optinist.utils import (
-    recursive_flatten_params,
-    split_dictionary,
-)
+from studio.app.optinist.wrappers.optinist.utils import recursive_flatten_params
 
 logger = AppLogger.get_logger()
 
@@ -80,22 +77,18 @@ def caiman_cnmf_multisession(
     if "dview" in locals():
         stop_server(dview=dview)  # noqa: F821
 
-    n_processes = None
+    # TODO: Add parameters for node
+    n_processes = 1
     dview = None
     # This process launches another process to run the CNMF algorithm,
     # so this node use at least 2 core.
-    # TODO: The minimum number of cores in the snakamake setting should be 2
-    if smk_parms["cores"] == 1:
-        c, dview, n_processes = setup_cluster(
-            backend="single", n_processes=smk_parms["cores"], single_thread=True
-        )
-    elif smk_parms["cores"] == 2:
+    if n_processes == 1:
         c, dview, n_processes = setup_cluster(
-            backend="single", n_processes=smk_parms["cores"] - 1, single_thread=True
+            backend="single", n_processes=n_processes, single_thread=True
         )
     else:
         c, dview, n_processes = setup_cluster(
-            backend="multiprocessing", n_processes=smk_parms["cores"] - 1
+            backend="multiprocessing", n_processes=n_processes
         )
     logger.info(f"n_processes: {n_processes}")
 

studio/app/optinist/wrappers/caiman/motion_correction.py

@@ -10,10 +10,7 @@
 from studio.app.common.dataclass import ImageData
 from studio.app.optinist.core.nwb.nwb import NWBDATASET
 from studio.app.optinist.dataclass import RoiData
-from studio.app.optinist.wrappers.optinist.utils import (
-    recursive_flatten_params,
-    split_dictionary,
-)
+from studio.app.optinist.wrappers.optinist.utils import recursive_flatten_params
 
 logger = AppLogger.get_logger()
 
@@ -29,9 +26,6 @@ def caiman_mc(
 
     function_id = ExptOutputPathIds(output_dir).function_id
     logger.info(f"start caiman motion_correction: {function_id}")
-    params, smk_parms = split_dictionary(
-        params, ["use_conda", "cores", "forceall", "forcetargets", "lock", "forcerun"]
-    )
     flattened_params = {}
     recursive_flatten_params(params, flattened_params)
     params = flattened_params
@@ -41,22 +35,18 @@ def caiman_mc(
     if params is not None:
         opts.change_params(params_dict=params)
 
-    n_processes = None
+    # TODO: Add parameters for node
+    n_processes = 1
     dview = None
     # This process launches another process to run the CNMF algorithm,
     # so this node use at least 2 core.
-    # TODO: The minimum number of cores in the snakamake setting should be 2
-    if smk_parms["cores"] == 1:
-        c, dview, n_processes = setup_cluster(
-            backend="single", n_processes=smk_parms["cores"], single_thread=True
-        )
-    elif smk_parms["cores"] == 2:
+    if n_processes == 1:
         c, dview, n_processes = setup_cluster(
-            backend="single", n_processes=smk_parms["cores"] - 1, single_thread=True
+            backend="single", n_processes=n_processes, single_thread=True
         )
     else:
         c, dview, n_processes = setup_cluster(
-            backend="multiprocessing", n_processes=smk_parms["cores"] - 1
+            backend="multiprocessing", n_processes=n_processes
         )
     logger.info(f"n_processes: {n_processes}")
 

studio/app/optinist/wrappers/lccd/lccd_detection.py

@@ -5,10 +5,7 @@
 from studio.app.common.dataclass import ImageData
 from studio.app.optinist.core.nwb.nwb import NWBDATASET
 from studio.app.optinist.dataclass import EditRoiData, FluoData, IscellData, RoiData
-from studio.app.optinist.wrappers.optinist.utils import (
-    recursive_flatten_params,
-    split_dictionary,
-)
+from studio.app.optinist.wrappers.optinist.utils import recursive_flatten_params
 
 logger = AppLogger.get_logger()
 
@@ -21,9 +18,6 @@ def lccd_detect(
     function_id = ExptOutputPathIds(output_dir).function_id
     logger.info("start lccd_detect: %s", function_id)
 
-    params, smk_parms = split_dictionary(
-        params, ["use_conda", "cores", "forceall", "forcetargets", "lock", "forcerun"]
-    )
     flattened_params = {}
     recursive_flatten_params(params, flattened_params)
     params = flattened_params

studio/app/optinist/wrappers/suite2p/registration.py

@@ -2,10 +2,7 @@
 from studio.app.common.core.logger import AppLogger
 from studio.app.common.dataclass import ImageData
 from studio.app.optinist.dataclass import Suite2pData
-from studio.app.optinist.wrappers.optinist.utils import (
-    recursive_flatten_params,
-    split_dictionary,
-)
+from studio.app.optinist.wrappers.optinist.utils import recursive_flatten_params
 
 logger = AppLogger.get_logger()
 
@@ -18,9 +15,6 @@ def suite2p_registration(
     function_id = ExptOutputPathIds(output_dir).function_id
     logger.info("start suite2p registration: %s", function_id)
 
-    params, _ = split_dictionary(
-        params, ["use_conda", "cores", "forceall", "forcetargets", "lock", "forcerun"]
-    )
     flattened_params = {}
     recursive_flatten_params(params, flattened_params)
     params = flattened_params

studio/app/optinist/wrappers/suite2p/roi.py

@@ -9,10 +9,7 @@
     RoiData,
     Suite2pData,
 )
-from studio.app.optinist.wrappers.optinist.utils import (
-    recursive_flatten_params,
-    split_dictionary,
-)
+from studio.app.optinist.wrappers.optinist.utils import recursive_flatten_params
 
 logger = AppLogger.get_logger()
 
@@ -26,9 +23,6 @@ def suite2p_roi(
     function_id = ExptOutputPathIds(output_dir).function_id
     logger.info("start suite2p_roi: %s", function_id)
 
-    params, _ = split_dictionary(
-        params, ["use_conda", "cores", "forceall", "forcetargets", "lock", "forcerun"]
-    )
     flattened_params = {}
     recursive_flatten_params(params, flattened_params)
     params = flattened_params

studio/app/optinist/wrappers/suite2p/spike_deconv.py

@@ -2,10 +2,7 @@
 from studio.app.common.core.logger import AppLogger
 from studio.app.optinist.core.nwb.nwb import NWBDATASET
 from studio.app.optinist.dataclass import SpikingActivityData, Suite2pData
-from studio.app.optinist.wrappers.optinist.utils import (
-    recursive_flatten_params,
-    split_dictionary,
-)
+from studio.app.optinist.wrappers.optinist.utils import recursive_flatten_params
 
 logger = AppLogger.get_logger()
 
@@ -19,9 +16,6 @@ def suite2p_spike_deconv(
     function_id = ExptOutputPathIds(output_dir).function_id
     logger.info("start suite2_spike_deconv: %s", function_id)
 
-    params, _ = split_dictionary(
-        params, ["use_conda", "cores", "forceall", "forcetargets", "lock", "forcerun"]
-    )
     flattened_params = {}
     recursive_flatten_params(params, flattened_params)
     params = flattened_params