Fixed translation for mapping multiple nested references

stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py

@@ -90,6 +90,18 @@ def get_first_of_type(itr, typ):
         """
         return TestAzureSentinelResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get('type') == typ)
 
+    @staticmethod
+    def get_n_of_type(itr, typ, n):
+        """
+        gets n's value from the list of respective stix objects
+        """
+
+        obj_list = [obj for obj in itr if isinstance(obj, dict) and obj.get('type') == typ]
+        if len(obj_list) >= n:
+            return obj_list[n-1]
+
+        return None
+
     @staticmethod
     def test_common_prop():
         """
@@ -133,6 +145,7 @@ def test_custom_property():
 
         x_msazure_sentinel = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-msazure-sentinel')
         x_msazure_sentinel_alert = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-msazure-sentinel-alert')
+        x_msazure_sentinel_alert2 = TestAzureSentinelResultsToStix.get_n_of_type(objects.values(), 'x-msazure-sentinel-alert', 2)
         x_ibm_finding = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-ibm-finding')
         x_oca_event = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-oca-event')
 
@@ -142,10 +155,12 @@ def test_custom_property():
         assert x_msazure_sentinel['subscription_id'] == '083de1fb-cd2d-4b7c-895a-2b5af1d091e8'
 
         assert x_msazure_sentinel_alert is not None, 'Custom object type not found'
+        assert x_msazure_sentinel_alert2 is not None, 'Custom object type not found'
 
-        assert x_msazure_sentinel_alert.keys() == {'type', 'recommendedactions', 'status', 'userStates'}
+        assert x_msazure_sentinel_alert.keys() == {'type', 'recommendedactions', 'status'}
+        assert x_msazure_sentinel_alert2.keys() == {'type', 'userStates'}
         assert type(x_msazure_sentinel_alert['recommendedactions']) is list
-        assert x_ibm_finding.keys() == {'type', 'createddatetime', 'description', 'time_observed', 'severity', 'name', 'src_os_ref'}
+        assert x_ibm_finding.keys() == {'type', 'createddatetime', 'description', 'time_observed', 'severity', 'name'}
         assert x_ibm_finding['name'] == 'Rare SVCHOST service group executed'
         assert x_oca_event.keys() == {'type', 'code', 'category', 'created', 'action'}
         assert x_oca_event['category'] == 'SuspiciousSVCHOSTRareGroup'

stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json

@@ -1587,7 +1587,7 @@
       {
         "key": "x-oca-asset.mac_refs",
         "object": "host",
-        "references": "host_mac",
+        "references": ["host_mac"],
         "unwrap": true
       }
     ],

stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py

@@ -3,7 +3,7 @@
 import uuid
 import json
 
-from stix_shifter_utils.utils.helpers import dict_merge
+from stix_shifter_utils.utils.helpers import dict_merge, remove_null, find, nested_set
 from stix_shifter_utils.stix_translation.src.json_to_stix import observable, id_contributing_properties
 from stix2validator import validate_instance, print_results, ValidationOptions
 from datetime import datetime
@@ -115,14 +115,14 @@ def _valid_stix_value(self, observable_key, stix_value):
             self.logger.debug("Failed to validate STIX property '{}' with value '{}'. Exception: {}".format(observable_key, stix_value, e))
             return False
 
-    def _compose_value_object(self, value, key_list, observable_key=None, object_tag_ref_map=None, transformer=None, references=None, unwrap=False):
+    def _compose_value_object(self, value, key_list, observable_key=None, object_tag_ref_map=None, transformer=None, references=None, unwrap=False, group=False, object_key_ind=None):
         """
         Converts the value of the data to STIX valid value
         """
         try:
             return_value = {}
             for key in key_list:
-                return_value[key] = self._compose_value_object(value, key_list[1:], observable_key=observable_key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap)
+                return_value[key] = self._compose_value_object(value, key_list[1:], observable_key=observable_key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind)
                 break
             else:
                 if transformer:
@@ -140,11 +140,17 @@ def _compose_value_object(self, value, key_list, observable_key=None, object_tag
                             if not isinstance(value, list):
                                 value = [value]
                             for i, _ in enumerate(value):
-                                parent_key_ind = self._get_tag_ind(ref, object_tag_ref_map, create_on_absence=False, unwrap=i)
-                                if parent_key_ind:
-                                    return_value.append(parent_key_ind)
+                                if isinstance(object_key_ind, int):
+                                    unwrap_ind = str(object_key_ind) + '_' + str(i+1)
+                                else:
+                                    unwrap_ind = i+1
+
+                                self._get_tag_ind(ref, object_tag_ref_map, create_on_absence=False, unwrap=unwrap_ind)
+
+                            if ref in object_tag_ref_map['refs']:
+                                return_value.extend(object_tag_ref_map['refs'][ref])
                     else:
-                        return_value = self._get_tag_ind(references, object_tag_ref_map, create_on_absence=False)
+                        return_value = self._get_tag_ind(references, object_tag_ref_map, create_on_absence=False, unwrap=object_key_ind)
                         # if the property has unwrap true and is not a list, convert to list
                         if unwrap is True and not isinstance(return_value, list):
                             return_value = [return_value]
@@ -153,26 +159,38 @@ def _compose_value_object(self, value, key_list, observable_key=None, object_tag
                         return None
                     return_value = value
 
+            if unwrap and isinstance(return_value, dict):
+                return_value = remove_null(return_value)
+
             return return_value
         except Exception as e:
             raise Exception("Error in json_to_stix_translator._compose_value_object: %s" % e)
 
 
-    def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap=False, property_key=None):
+    def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap=False, property_key=None, ref_tag=None):
         """
         Gets the stringified index of the observable object from the `object_tag_ref_map` cached dictionary if it exists
         or creates otherwise.
         """
         tag_ind = None
         tag_ind_str = None
         try:
+            if not ref_tag:
+                ref_tag = tag
+
             # if the datasource fields is a collection of json object than we need to unwrap it and create multiple objects
             if unwrap:
                 tag = tag + '_' + str(unwrap)
 
             if tag in object_tag_ref_map['tags']:
                 tag_ind = object_tag_ref_map['tags'][tag]['i']
                 object_tag_ref_map['tags'][tag]['n'] += 1
+
+                if ref_tag in object_tag_ref_map['refs'] and tag_ind not in object_tag_ref_map['refs'][ref_tag]:
+                    object_tag_ref_map['refs'][ref_tag].append(tag_ind)
+                    if not self.spec_version == "2.1":
+                        object_tag_ref_map['refs'][ref_tag].sort()
+
             elif create_on_absence:
                 tag_ind_str = str(object_tag_ref_map[UUID5_NAMESPACE])
                 if self.spec_version == "2.1":
@@ -182,6 +200,7 @@ def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap=
 
                 object_tag_ref_map[UUID5_NAMESPACE] += 1
                 object_tag_ref_map['tags'][tag] = {'i': tag_ind, 'n': 0}
+                object_tag_ref_map['refs'][ref_tag] = [tag_ind]
 
             if tag_ind is not None:
                 if not tag_ind_str:
@@ -234,7 +253,7 @@ def _handle_properties(self, to_stix_config_prop, data, objects, object_tag_ref_
                 elif isinstance(data, list):
                     for i, d in enumerate(data):
                         if isinstance(d, list) or isinstance(d, dict):
-                            self._handle_properties(to_stix_config_prop, d, objects, object_tag_ref_map, data, ds_sub_key, i)
+                            self._handle_properties(to_stix_config_prop, d, objects, object_tag_ref_map, data, ds_sub_key, i+1)
                         else:
                             # data variable is the final value, process in bulk
                             self._handle_value(data, parent_data, ds_sub_key, to_stix_config_prop, objects, object_tag_ref_map, object_key_ind)
@@ -276,6 +295,7 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje
                 # unwrap array of stix values to separate stix objects
                 unwrap = True if 'unwrap' in prop and isinstance(data, list) else False
                 cybox = prop.get('cybox', self.cybox_default)
+                group = prop['group'] if 'group' in prop else False
 
                 if self.callback:
                     try:
@@ -287,23 +307,22 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje
 
                 config_keys = key.split('.')
                 if len(config_keys) < 2:
-                    if False is prop.get('cybox', self.cybox_default): 
-                        object_tag_ref_map['out_cybox'][key] = self._compose_value_object(data, [], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap)
+                    if False is cybox: 
+                        object_tag_ref_map['out_cybox'][key] = self._compose_value_object(data, [], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind)
                     pass
                 else:
                     type_name = config_keys[0]
                     property_key = config_keys[1]
                     parent_key = prop['object'] if 'object' in prop else type_name
-
-                    group = prop['group'] if 'group' in prop else False
                     substitute_key = prop['ds_key'] if 'ds_key' in prop else None
+                    ref_tag = parent_key
 
                     if False is cybox and not substitute_key:
-                        value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap)
+                        value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind)
                         self._add_property(type_name, property_key, type_name, value, object_tag_ref_map['out_cybox'], cybox=False)
                         continue
 
-                    if object_key_ind:
+                    if isinstance(object_key_ind, int):
                         parent_key = parent_key + '_' + str(object_key_ind)
 
                     # use the hard-coded value in the mapping
@@ -316,17 +335,23 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje
                             if False is cybox:
                                 object_tag_ref_map['ds_key_cybox'][substitute_key] = True
 
-                        value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap)
+                        value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind)
 
                     if value is None or value == '':
                         continue
 
                     if not references and unwrap and isinstance(value, list):
                         for i, val_el in enumerate(value):
-                            parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i, property_key=property_key)
+                            parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key, ref_tag=ref_tag)
                             self._add_property(type_name, property_key, parent_key_ind, val_el, objects, group=group)
+                    elif not references and unwrap and isinstance(data, list):
+                        enum_value = find('.'.join(config_keys[2:]), value)
+                        for i, val_el in enumerate(enum_value):
+                            val_el_new = nested_set(value, config_keys[2:], val_el)
+                            parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key, ref_tag=ref_tag)
+                            self._add_property(type_name, property_key, parent_key_ind, val_el_new, objects, group=group)
                     else:
-                        parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, property_key=property_key)
+                        parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, property_key=property_key, ref_tag=ref_tag)
                         self._add_property(type_name, property_key, parent_key_ind, value, objects, group=group)
         except Exception as e:
             raise Exception("Error in json_to_stix_translator._handle_value: %s : %s" % (e, e.__traceback__.tb_lineno))
@@ -384,7 +409,7 @@ def transform(self, obj):
 
             # create normal type objects
             if isinstance(obj, dict):
-                object_tag_ref_map = {UUID5_NAMESPACE: 0, 'tags': {}, 'non_ref_props': {}, 'out_cybox': {}, 'ds_key_cybox': {}}
+                object_tag_ref_map = {UUID5_NAMESPACE: 0, 'tags': {}, 'refs': {}, 'non_ref_props': {}, 'out_cybox': {}, 'ds_key_cybox': {}}
 
                 self._handle_properties(ds_map, obj, object_map, object_tag_ref_map)
                 # special case:
@@ -395,6 +420,8 @@ def transform(self, obj):
 
                 for k, v in object_tag_ref_map['out_cybox'].items():
                     observation[k] = v
+
+                # print(json.dumps(object_tag_ref_map, cls=StixObjectIdEncoder))
 
             else:
                 self.logger.debug("Not a dict: {}".format(obj))

stix_shifter_utils/utils/helpers.py

@@ -91,3 +91,29 @@ def find(element, dd, default=None):
         return rv
     except Exception:
         return default
+
+
+def remove_null(d):
+    clean = {}   
+    for k, v in d.items():
+        if isinstance(v, dict):
+            nested = remove_null(v)
+            if nested and len(nested.keys()) > 0:
+                clean[k] = nested
+        elif v:
+            clean[k] = v
+
+    if clean == {}:
+        clean = None
+    return clean
+
+
+def nested_set(dic, keys, value):
+    """Set item in nested dictionary"""
+    from copy import deepcopy
+    from functools import reduce
+    from operator import getitem
+
+    new_dic = deepcopy(dic)
+    reduce(getitem, keys[:-1], new_dic)[keys[-1]] = value
+    return new_dic