[15.0][FWD] connector_importer: misc fix/imp

connector_importer/components/__init__.py

@@ -7,3 +7,4 @@
 from . import mapper
 from . import automapper
 from . import dynamicmapper
+from . import listeners

connector_importer/components/dynamicmapper.py

@@ -34,14 +34,14 @@ def dynamic_fields(self, record):
                 # Never convert IDs
                 continue
             fname = source_fname
-            if prefix and source_fname.startswith(prefix):
-                # Eg: prefix all supplier fields w/ `supplier_`
-                fname = fname[len(prefix) :]
-                clean_record[fname] = clean_record.pop(source_fname)
             if "::" in fname:
                 # Eg: transformers like `xid::``
                 fname = fname.split("::")[-1]
                 clean_record[fname] = clean_record.pop(source_fname)
+            if prefix and fname.startswith(prefix):
+                # Eg: prefix all supplier fields w/ `supplier.`
+                fname = fname[len(prefix) :]
+                clean_record[fname] = clean_record.pop(prefix + fname)
             if available_fields.get(fname):
                 fspec = available_fields.get(fname)
                 ftype = fspec["type"]
@@ -78,10 +78,13 @@ def _get_valid_keys(self, record):
         valid_keys = [k for k in record.keys() if not k.startswith("_")]
         prefix = self._source_key_prefix
         if prefix:
-            valid_keys = [k for k in valid_keys if k.startswith(prefix)]
+            valid_keys = [k for k in valid_keys if prefix in k]
         whitelist = self._source_key_whitelist
         if whitelist:
             valid_keys = [k for k in valid_keys if k in whitelist]
+        blacklist = self._source_key_blacklist
+        if blacklist:
+            valid_keys = [k for k in valid_keys if k not in blacklist]
         return tuple(valid_keys)
 
     def _required_keys(self):
@@ -91,6 +94,10 @@ def _required_keys(self):
     def _source_key_whitelist(self):
         return self.work.options.mapper.get("source_key_whitelist", [])
 
+    @property
+    def _source_key_blacklist(self):
+        return self.work.options.mapper.get("source_key_blacklist", [])
+
     @property
     def _source_key_empty_skip(self):
         """List of source keys to skip when empty.
@@ -106,38 +113,32 @@ def _source_key_empty_skip(self):
     def _source_key_prefix(self):
         return self.work.options.mapper.get("source_key_prefix", "")
 
-    @property
-    def _source_key_xid_module(self):
-        """Module name to use to sanitize XMLids"""
-        return self.work.options.mapper.get("source_key_xid_module", "")
-
     def _is_xmlid_key(self, fname, ftype):
         return fname.startswith("xid::") and ftype in (
             "many2one",
             "one2many",
             "many2many",
         )
 
-    def _dynamic_keys_mapping(self, fname):
+    def _dynamic_keys_mapping(self, fname, **options):
         return {
             "char": lambda self, rec, fname: rec[fname],
             "text": lambda self, rec, fname: rec[fname],
             "selection": lambda self, rec, fname: rec[fname],
-            "integer": convert(fname, "safe_int"),
-            "float": convert(fname, "safe_float"),
-            "boolean": convert(fname, "bool"),
-            "date": convert(fname, "date"),
-            "datetime": convert(fname, "utc_date"),
-            "many2one": backend_to_rel(fname),
-            "many2many": backend_to_rel(fname),
-            "one2many": backend_to_rel(fname),
-            "_xmlid": xmlid_to_rel(
-                fname, sanitize_default_mod_name=self._source_key_xid_module
-            ),
+            "integer": convert(fname, "safe_int", **options),
+            "float": convert(fname, "safe_float", **options),
+            "boolean": convert(fname, "bool", **options),
+            "date": convert(fname, "date", **options),
+            "datetime": convert(fname, "utc_date", **options),
+            "many2one": backend_to_rel(fname, **options),
+            "many2many": backend_to_rel(fname, **options),
+            "one2many": backend_to_rel(fname, **options),
+            "_xmlid": xmlid_to_rel(fname, **options),
         }
 
     def _get_converter(self, fname, ftype):
-        return self._dynamic_keys_mapping(fname).get(ftype)
+        options = self.work.options.mapper.get("converter", {}).get(fname, {})
+        return self._dynamic_keys_mapping(fname, **options).get(ftype)
 
     _non_mapped_keys_cache = None
 

connector_importer/components/importer.py

@@ -295,13 +295,14 @@ def _load_mapper_options(self):
         """Retrieve mapper options."""
         return {"override_existing": self.must_override_existing}
 
-    # TODO: make these contexts customizable via recordset settings
     def _odoo_default_context(self):
         """Default context to be used in both create and write methods"""
-        return {
+        ctx = {
             "importer_type_id": self.recordset.import_type_id.id,
             "tracking_disable": True,
         }
+        ctx.update(self.work.options.importer.get("ctx", {}))
+        return ctx
 
     def _odoo_create_context(self):
         """Inject context variables on create, merged by odoorecord handler."""
@@ -394,17 +395,32 @@ def run(self, record, is_last_importer=True, **kw):
             ]
         ).format(**counters)
         self.tracker._log(msg)
-        self._trigger_finish_events(record, is_last_importer=is_last_importer)
+        self.finalize_session(record, is_last_importer=is_last_importer)
         return counters
 
-    def _trigger_finish_events(self, record, is_last_importer=False):
-        """Trigger events when the importer has done its job."""
+    def finalize_session(self, record, is_last_importer=False):
+        self._trigger_importer_events(record)
         if is_last_importer:
-            # Trigger global event for recordset
-            self.recordset._event(
-                "on_last_record_import_finished", collection=self.work.collection
-            ).notify(self, record)
-            # Trigger model specific event
-            self.model.browse()._event(
-                "on_last_record_import_finished", collection=self.work.collection
-            ).notify(self, record)
+            self._trigger_finish_events(record)
+
+    def _trigger_importer_events(self, record):
+        """Trigger events when the importer has done its job."""
+        # Trigger global event for recordset
+        self.recordset._event(
+            "on_record_import_finished", collection=self.work.collection
+        ).notify(self, record)
+        # Trigger model specific event
+        self.model.browse()._event(
+            "on_record_import_finished", collection=self.work.collection
+        ).notify(self, record)
+
+    def _trigger_finish_events(self, record):
+        """Trigger events when the importer has done its job."""
+        # Trigger global event for recordset
+        self.recordset._event(
+            "on_last_record_import_finished", collection=self.work.collection
+        ).notify(self, record)
+        # Trigger model specific event
+        self.model.browse()._event(
+            "on_last_record_import_finished", collection=self.work.collection
+        ).notify(self, record)

connector_importer/components/listeners.py

@@ -0,0 +1,81 @@
+# Copyright 2023 Camptocamp SA
+# @author: Simone Orsi <[email protected]>
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
+from functools import partial
+
+from odoo.addons.component.core import Component
+
+
+class ImportRecordsetEventListener(Component):
+    _name = "recordset.event.listener"
+    _inherit = "base.connector.listener"
+    _apply_on = ["import.recordset"]
+
+    def on_last_record_import_finished(self, importer, record):
+        if self._must_run_server_action(importer, record, "last_importer_done"):
+            self._run_server_actions(importer, record)
+
+    def on_record_import_finished(self, importer, record):
+        if self._must_run_server_action(importer, record, "each_importer_done"):
+            self._run_server_actions(importer, record)
+
+    def _must_run_server_action(self, importer, record, trigger):
+        recordset = record.recordset_id
+        return bool(
+            recordset.server_action_ids
+            and recordset.server_action_trigger_on == trigger
+            and self._has_records_to_process(importer)
+        )
+
+    def _has_records_to_process(self, importer):
+        counters = importer.tracker.get_counters()
+        return counters["created"] or counters["updated"]
+
+    def _run_server_actions(self, importer, record):
+        """Execute one or more server actions tied to the recordset."""
+        recordset = record.recordset_id
+        actions = recordset.server_action_ids
+        report_by_model = recordset.get_report_by_model()
+        # execute actions by importer order
+        for model, report in report_by_model.items():
+            action = actions.filtered(lambda x: x.model_id == model)
+            if not action:
+                continue
+            record_ids = sorted(set(report["created"] + report["updated"]))
+            if not record_ids:
+                continue
+            self._add_after_commit_hook(recordset.id, action.id, record_ids)
+        generic_action = actions.filtered(
+            lambda x: x.model_id.model == "import.recordset"
+        )
+        if generic_action:
+            self._add_after_commit_hook(recordset.id, generic_action.id, recordset.ids)
+
+    def _run_server_action(self, recordset_id, action_id, record_ids):
+        action = self.env["ir.actions.server"].browse(action_id)
+        action = action.with_context(
+            **self._run_server_action_ctx(recordset_id, action_id, record_ids)
+        )
+        return action.run()
+
+    def _run_server_action_ctx(self, recordset_id, action_id, record_ids):
+        action = self.env["ir.actions.server"].browse(action_id)
+        action_ctx = dict(
+            active_model=action.model_id.model, import_recordset_id=recordset_id
+        )
+        if len(record_ids) > 1:
+            action_ctx["active_ids"] = record_ids
+        else:
+            action_ctx["active_id"] = record_ids[0]
+        return action_ctx
+
+    def _add_after_commit_hook(self, recordset_id, action_id, record_ids):
+        self.env.cr.postcommit.add(
+            partial(
+                self._run_server_action_post_commit, recordset_id, action_id, record_ids
+            ),
+        )
+
+    def _run_server_action_post_commit(self, recordset_id, action_id, record_ids):
+        self._run_server_action(recordset_id, action_id, record_ids)
+        self.env.cr.commit()  # pylint: disable=invalid-commit

connector_importer/components/odoorecord.py

@@ -2,6 +2,8 @@
 # Copyright 2018 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
 
+from odoo.tools import safe_eval
+
 from odoo.addons.component.core import Component
 
 from ..utils.misc import sanitize_external_id
@@ -38,33 +40,68 @@
 
     def odoo_find_domain(self, values, orig_values):
         """Domain to find the record in odoo."""
+        domain = self._odoo_find_domain_from_options(values, orig_values)
+        if not domain:
+            if not self.unique_key:
+                raise ValueError("No unique key and no domain to find this record")
+            domain = self._odoo_find_domain_from_unique_key(values, orig_values)
+        return domain
+
+    def _odoo_find_domain_from_options(self, values, orig_values):
+        """Evaluate domain from options if any."""
+        match_domain = self.work.options.record_handler.match_domain
+        if not match_domain:
+            return []
+        eval_ctx = self._domain_from_options_eval_ctx(values, orig_values)
+        domain = safe_eval.safe_eval(
+            self.work.options.record_handler.match_domain, eval_ctx
+        )
+        if not isinstance(domain, list):
+            raise ValueError("match_domain must be a list")
+        return domain
+
+    def _domain_from_options_eval_ctx(self, values, orig_values):
+        return {
+            "env": self.env,
+            "user": self.env.user,
+            "datetime": safe_eval.datetime,
+            "dateutil": safe_eval.dateutil,
+            "time": safe_eval.time,
+            "values": values,
+            "orig_values": orig_values,
+            "ref_id": lambda x: self._smart_ref(x).id,
+            "ref": lambda x: self._smart_ref(x),
+        }
+
+    def _odoo_find_domain_from_unique_key(self, values, orig_values):
         value = NO_VALUE
         if self.unique_key in values:
             value = values[self.unique_key]
         elif self.unique_key in orig_values:
             value = orig_values[self.unique_key]
         if value is NO_VALUE:
             raise ValueError(
-                f"Cannot find {self.unique_key} in `values` nor `orig_values`"
+                f"Cannot find `{self.unique_key}` key in `values` nor `orig_values`"
             )
         return [(self.unique_key, "=", value)]
 
     def odoo_find(self, values, orig_values):
         """Find any existing item in odoo."""
-        if self.unique_key == "":
+        if self.unique_key and self.unique_key_is_xmlid:
             # if unique_key is None we might use as special find domain
-            return self.model
-        if self.unique_key_is_xmlid:
             xid = self._get_xmlid(values, orig_values)
             item = self.env.ref(xid, raise_if_not_found=False)
-            return item
+            return item or self.model
         item = self.model.search(
             self.odoo_find_domain(values, orig_values),
             order="create_date desc",
             limit=1,
         )
         return item
 
+    def _smart_ref(self, xid):
+        return self.env.ref(sanitize_external_id(xid))
+
     def _get_xmlid(self, values, orig_values):
         # Mappers will remove `xid::` prefix from the final values
         # hence, look for the original key.

connector_importer/models/import_type.py

@@ -63,7 +63,7 @@
 
     name = fields.Char(required=True, help="A meaningful human-friendly name")
     description = fields.Text()
-    key = fields.Char(required=True, help="Unique mnemonic identifier")
+    key = fields.Char(required=True, help="Unique mnemonic identifier", copy=False)
     options = fields.Text(help="YAML configuration")
     settings = fields.Text(
         string="Legacy Settings",
@@ -167,3 +167,9 @@
                 if _line == lines[-1]:
                     is_last_importer = True
                 yield (model_name.strip(), importer.strip(), is_last_importer)
+
+    def copy_data(self, default=None):
+        res = super().copy_data(default)
+        for data, rec in zip(res, self):
+            data["key"] = rec.key + "_COPY_FIXME"
+        return res