From 114d4dbf68966a4e8f6cbbc9f58e1e04a980b57c Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Mon, 8 Aug 2022 19:03:49 -0300 Subject: [PATCH 1/3] Fixed translation for mapping multiple nested references --- .../json_to_stix/json_to_stix_translator.py | 42 +++++++++++++------ stix_shifter_utils/utils/helpers.py | 26 ++++++++++++ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py index bdbe9b6e6..24af56954 100644 --- a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py @@ -2,7 +2,7 @@ import uuid import json -from stix_shifter_utils.utils.helpers import dict_merge +from stix_shifter_utils.utils.helpers import dict_merge, remove_null, find, nested_set from stix_shifter_utils.stix_translation.src.json_to_stix import observable from stix2validator import validate_instance, print_results, ValidationOptions from datetime import datetime @@ -115,14 +115,14 @@ def _valid_stix_value(self, observable_key, stix_value): self.logger.debug("Failed to validate STIX property '{}' with value '{}'. Exception: {}".format(observable_key, stix_value, e)) return False - def _compose_value_object(self, value, key_list, observable_key=None, object_tag_ref_map=None, transformer=None, references=None, unwrap=False): + def _compose_value_object(self, value, key_list, observable_key=None, object_tag_ref_map=None, transformer=None, references=None, unwrap=False, group=False, object_key_ind=None): """ Converts the value of the data to STIX valid value """ try: return_value = {} for key in key_list: - return_value[key] = self._compose_value_object(value, key_list[1:], observable_key=observable_key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap) + return_value[key] = self._compose_value_object(value, key_list[1:], observable_key=observable_key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind) break else: if transformer: @@ -140,7 +140,12 @@ def _compose_value_object(self, value, key_list, observable_key=None, object_tag if not isinstance(value, list): value = [value] for i, _ in enumerate(value): - parent_key_ind = self._get_tag_ind(ref, object_tag_ref_map, create_on_absence=False, unwrap=i) + if isinstance(object_key_ind, int): + unwrap_ind = str(object_key_ind) + '_' + str(i+1) + else: + unwrap_ind = i+1 + + parent_key_ind = self._get_tag_ind(ref, object_tag_ref_map, create_on_absence=False, unwrap=unwrap_ind) if parent_key_ind: return_value.append(parent_key_ind) else: @@ -153,6 +158,10 @@ def _compose_value_object(self, value, key_list, observable_key=None, object_tag return None return_value = value + # if unwrap and isinstance(return_value, dict): + # print( key_list, unwrap, references, observable_key, return_value) + # return_value = remove_null(return_value) + return return_value except Exception as e: raise Exception("Error in json_to_stix_translator._compose_value_object: %s" % e) @@ -234,7 +243,7 @@ def _handle_properties(self, to_stix_config_prop, data, objects, object_tag_ref_ elif isinstance(data, list): for i, d in enumerate(data): if isinstance(d, list) or isinstance(d, dict): - self._handle_properties(to_stix_config_prop, d, objects, object_tag_ref_map, data, ds_sub_key, i) + self._handle_properties(to_stix_config_prop, d, objects, object_tag_ref_map, data, ds_sub_key, i+1) else: # data variable is the final value, process in bulk self._handle_value(data, parent_data, ds_sub_key, to_stix_config_prop, objects, object_tag_ref_map, object_key_ind) @@ -276,6 +285,7 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje # unwrap array of stix values to separate stix objects unwrap = True if 'unwrap' in prop and isinstance(data, list) else False cybox = prop.get('cybox', self.cybox_default) + group = prop['group'] if 'group' in prop else False if self.callback: try: @@ -287,23 +297,21 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje config_keys = key.split('.') if len(config_keys) < 2: - if False is prop.get('cybox', self.cybox_default): - object_tag_ref_map['out_cybox'][key] = self._compose_value_object(data, [], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap) + if False is cybox: + object_tag_ref_map['out_cybox'][key] = self._compose_value_object(data, [], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind) pass else: type_name = config_keys[0] property_key = config_keys[1] parent_key = prop['object'] if 'object' in prop else type_name - - group = prop['group'] if 'group' in prop else False substitute_key = prop['ds_key'] if 'ds_key' in prop else None if False is cybox and not substitute_key: - value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap) + value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind) self._add_property(type_name, property_key, type_name, value, object_tag_ref_map['out_cybox'], cybox=False) continue - if object_key_ind: + if isinstance(object_key_ind, int): parent_key = parent_key + '_' + str(object_key_ind) # use the hard-coded value in the mapping @@ -316,15 +324,21 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje if False is cybox: object_tag_ref_map['ds_key_cybox'][substitute_key] = True - value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap) + value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind) if value is None or value == '': continue if not references and unwrap and isinstance(value, list): for i, val_el in enumerate(value): - parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i, property_key=property_key) + parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key) self._add_property(type_name, property_key, parent_key_ind, val_el, objects, group=group) + elif not references and unwrap and isinstance(data, list): + enum_value = find('.'.join(config_keys[2:]), value) + for i, val_el in enumerate(enum_value): + val_el_new = nested_set(value, config_keys[2:], val_el) + parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key) + self._add_property(type_name, property_key, parent_key_ind, val_el_new, objects, group=group) else: parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, property_key=property_key) self._add_property(type_name, property_key, parent_key_ind, value, objects, group=group) @@ -395,6 +409,8 @@ def transform(self, obj): for k, v in object_tag_ref_map['out_cybox'].items(): observation[k] = v + + # print(json.dumps(object_tag_ref_map, cls=StixObjectIdEncoder)) else: self.logger.debug("Not a dict: {}".format(obj)) diff --git a/stix_shifter_utils/utils/helpers.py b/stix_shifter_utils/utils/helpers.py index 8aefbf5a5..dc1b81aa0 100644 --- a/stix_shifter_utils/utils/helpers.py +++ b/stix_shifter_utils/utils/helpers.py @@ -94,3 +94,29 @@ def find(element, dd, default=None): return rv except Exception: return default + + +def remove_null(d): + clean = {} + for k, v in d.items(): + if isinstance(v, dict): + nested = remove_null(v) + if nested and len(nested.keys()) > 0: + clean[k] = nested + elif v: + clean[k] = v + + if clean == {}: + clean = None + return clean + + +def nested_set(dic, keys, value): + """Set item in nested dictionary""" + from copy import deepcopy + from functools import reduce + from operator import getitem + + new_dic = deepcopy(dic) + reduce(getitem, keys[:-1], new_dic)[keys[-1]] = value + return new_dic From 7acb683a60431b5a0a4afec9c6286867c164ece7 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Thu, 8 Sep 2022 12:59:41 -0400 Subject: [PATCH 2/3] Fixed translation for mapping multiple nested references 2 --- .../json_to_stix/json_to_stix_translator.py | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py index 24af56954..2e433bb6a 100644 --- a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py @@ -145,9 +145,10 @@ def _compose_value_object(self, value, key_list, observable_key=None, object_tag else: unwrap_ind = i+1 - parent_key_ind = self._get_tag_ind(ref, object_tag_ref_map, create_on_absence=False, unwrap=unwrap_ind) - if parent_key_ind: - return_value.append(parent_key_ind) + self._get_tag_ind(ref, object_tag_ref_map, create_on_absence=False, unwrap=unwrap_ind) + + if ref in object_tag_ref_map['refs']: + return_value = object_tag_ref_map['refs'][ref] else: return_value = self._get_tag_ind(references, object_tag_ref_map, create_on_absence=False) # if the property has unwrap true and is not a list, convert to list @@ -158,16 +159,15 @@ def _compose_value_object(self, value, key_list, observable_key=None, object_tag return None return_value = value - # if unwrap and isinstance(return_value, dict): - # print( key_list, unwrap, references, observable_key, return_value) - # return_value = remove_null(return_value) + if unwrap and isinstance(return_value, dict): + return_value = remove_null(return_value) return return_value except Exception as e: raise Exception("Error in json_to_stix_translator._compose_value_object: %s" % e) - def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap=False, property_key=None): + def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap=False, property_key=None, ref_tag=None): """ Gets the stringified index of the observable object from the `object_tag_ref_map` cached dictionary if it exists or creates otherwise. @@ -175,6 +175,9 @@ def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap= tag_ind = None tag_ind_str = None try: + if not ref_tag: + ref_tag = tag + # if the datasource fields is a collection of json object than we need to unwrap it and create multiple objects if unwrap: tag = tag + '_' + str(unwrap) @@ -182,6 +185,12 @@ def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap= if tag in object_tag_ref_map['tags']: tag_ind = object_tag_ref_map['tags'][tag]['i'] object_tag_ref_map['tags'][tag]['n'] += 1 + + if ref_tag in object_tag_ref_map['refs'] and tag_ind not in object_tag_ref_map['refs'][ref_tag]: + object_tag_ref_map['refs'][ref_tag].append(tag_ind) + if not self.spec_version == "2.1": + object_tag_ref_map['refs'][ref_tag].sort() + elif create_on_absence: tag_ind_str = str(object_tag_ref_map[UUID5_NAMESPACE]) if self.spec_version == "2.1": @@ -191,6 +200,7 @@ def _get_tag_ind(self, tag, object_tag_ref_map, create_on_absence=False, unwrap= object_tag_ref_map[UUID5_NAMESPACE] += 1 object_tag_ref_map['tags'][tag] = {'i': tag_ind, 'n': 0} + object_tag_ref_map['refs'][ref_tag] = [tag_ind] if tag_ind is not None: if not tag_ind_str: @@ -305,6 +315,7 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje property_key = config_keys[1] parent_key = prop['object'] if 'object' in prop else type_name substitute_key = prop['ds_key'] if 'ds_key' in prop else None + ref_tag = parent_key if False is cybox and not substitute_key: value = self._compose_value_object(data, config_keys[2:], observable_key=key, object_tag_ref_map=object_tag_ref_map, transformer=transformer, references=references, unwrap=unwrap, group=group, object_key_ind=object_key_ind) @@ -331,16 +342,16 @@ def _handle_value(self, data, parent_data, ds_sub_key, to_stix_config_prop, obje if not references and unwrap and isinstance(value, list): for i, val_el in enumerate(value): - parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key) + parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key, ref_tag=ref_tag) self._add_property(type_name, property_key, parent_key_ind, val_el, objects, group=group) elif not references and unwrap and isinstance(data, list): enum_value = find('.'.join(config_keys[2:]), value) for i, val_el in enumerate(enum_value): val_el_new = nested_set(value, config_keys[2:], val_el) - parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key) + parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, unwrap=i+1, property_key=property_key, ref_tag=ref_tag) self._add_property(type_name, property_key, parent_key_ind, val_el_new, objects, group=group) else: - parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, property_key=property_key) + parent_key_ind = self._get_tag_ind(parent_key, object_tag_ref_map, create_on_absence=True, property_key=property_key, ref_tag=ref_tag) self._add_property(type_name, property_key, parent_key_ind, value, objects, group=group) except Exception as e: raise Exception("Error in json_to_stix_translator._handle_value: %s : %s" % (e, e.__traceback__.tb_lineno)) @@ -398,7 +409,7 @@ def transform(self, obj): # create normal type objects if isinstance(obj, dict): - object_tag_ref_map = {UUID5_NAMESPACE: 0, 'tags': {}, 'non_ref_props': {}, 'out_cybox': {}, 'ds_key_cybox': {}} + object_tag_ref_map = {UUID5_NAMESPACE: 0, 'tags': {}, 'refs': {}, 'non_ref_props': {}, 'out_cybox': {}, 'ds_key_cybox': {}} self._handle_properties(ds_map, obj, object_map, object_tag_ref_map) # special case: From dfb9d59579ae916893a0ba2cea62c2c09f902e28 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Thu, 8 Sep 2022 16:42:39 -0400 Subject: [PATCH 3/3] Fixed translation for mapping multiple nested references unit tests --- .../test_azure_sentinel_json_to_stix.py | 19 +++++++++++++++++-- .../stix_translation/json/to_stix_map.json | 2 +- .../json_to_stix/json_to_stix_translator.py | 4 ++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py b/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py index f3876bafd..5f480e2c5 100644 --- a/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py +++ b/stix_shifter_modules/azure_sentinel/tests/stix_translation/test_azure_sentinel_json_to_stix.py @@ -90,6 +90,18 @@ def get_first_of_type(itr, typ): """ return TestAzureSentinelResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get('type') == typ) + @staticmethod + def get_n_of_type(itr, typ, n): + """ + gets n's value from the list of respective stix objects + """ + + obj_list = [obj for obj in itr if isinstance(obj, dict) and obj.get('type') == typ] + if len(obj_list) >= n: + return obj_list[n-1] + + return None + @staticmethod def test_common_prop(): """ @@ -133,6 +145,7 @@ def test_custom_property(): x_msazure_sentinel = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-msazure-sentinel') x_msazure_sentinel_alert = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-msazure-sentinel-alert') + x_msazure_sentinel_alert2 = TestAzureSentinelResultsToStix.get_n_of_type(objects.values(), 'x-msazure-sentinel-alert', 2) x_ibm_finding = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-ibm-finding') x_oca_event = TestAzureSentinelResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') @@ -142,10 +155,12 @@ def test_custom_property(): assert x_msazure_sentinel['subscription_id'] == '083de1fb-cd2d-4b7c-895a-2b5af1d091e8' assert x_msazure_sentinel_alert is not None, 'Custom object type not found' + assert x_msazure_sentinel_alert2 is not None, 'Custom object type not found' - assert x_msazure_sentinel_alert.keys() == {'type', 'recommendedactions', 'status', 'userStates'} + assert x_msazure_sentinel_alert.keys() == {'type', 'recommendedactions', 'status'} + assert x_msazure_sentinel_alert2.keys() == {'type', 'userStates'} assert type(x_msazure_sentinel_alert['recommendedactions']) is list - assert x_ibm_finding.keys() == {'type', 'createddatetime', 'description', 'time_observed', 'severity', 'name', 'src_os_ref'} + assert x_ibm_finding.keys() == {'type', 'createddatetime', 'description', 'time_observed', 'severity', 'name'} assert x_ibm_finding['name'] == 'Rare SVCHOST service group executed' assert x_oca_event.keys() == {'type', 'code', 'category', 'created', 'action'} assert x_oca_event['category'] == 'SuspiciousSVCHOSTRareGroup' diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json index 066f5e2e0..27eeb93da 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json @@ -1587,7 +1587,7 @@ { "key": "x-oca-asset.mac_refs", "object": "host", - "references": "host_mac", + "references": ["host_mac"], "unwrap": true } ], diff --git a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py index 2e433bb6a..a9abfa640 100644 --- a/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py +++ b/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py @@ -148,9 +148,9 @@ def _compose_value_object(self, value, key_list, observable_key=None, object_tag self._get_tag_ind(ref, object_tag_ref_map, create_on_absence=False, unwrap=unwrap_ind) if ref in object_tag_ref_map['refs']: - return_value = object_tag_ref_map['refs'][ref] + return_value.extend(object_tag_ref_map['refs'][ref]) else: - return_value = self._get_tag_ind(references, object_tag_ref_map, create_on_absence=False) + return_value = self._get_tag_ind(references, object_tag_ref_map, create_on_absence=False, unwrap=object_key_ind) # if the property has unwrap true and is not a list, convert to list if unwrap is True and not isinstance(return_value, list): return_value = [return_value]