diff --git a/setup.cfg b/setup.cfg index f3c12071..e2b53539 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ install_requires = docker>=5.0.0 stix-shifter>=5.3.0 stix-shifter-utils>=5.3.0 - firepit>=2.3.20 + firepit>=2.3.21 typeguard tests_require = pytest diff --git a/src/kestrel/codegen/commands.py b/src/kestrel/codegen/commands.py index b1136006..9cb0ce37 100644 --- a/src/kestrel/codegen/commands.py +++ b/src/kestrel/codegen/commands.py @@ -111,10 +111,14 @@ def wrapper(stmt, session): @_default_output def assign(stmt, session): entity_table = session.symtable[stmt["input"]].entity_table - transform = stmt.get("transform") + transform = stmt.get("transformer") if transform: if transform.lower() == "timestamped": qry = session.store.timestamped(entity_table, run=False) + elif transform.lower() == "addobsid": + qry = session.store.extract_observeddata_attribute( + entity_table, name_of_attribute="id", run=False + ) else: qry = Query(entity_table) else: @@ -214,10 +218,14 @@ def info(stmt, session): @_debug_logger def disp(stmt, session): entity_table = session.symtable[stmt["input"]].entity_table - transform = stmt.get("transform") + transform = stmt.get("transformer") if transform and entity_table: if transform.lower() == "timestamped": qry = session.store.timestamped(entity_table, run=False) + elif transform.lower() == "addobsid": + qry = session.store.extract_observeddata_attribute( + entity_table, name_of_attribute="id", run=False + ) else: qry = Query(entity_table) else: diff --git a/src/kestrel/semantics/completor.py b/src/kestrel/semantics/completor.py index ea2fe8c8..556cd2a0 100644 --- a/src/kestrel/semantics/completor.py +++ b/src/kestrel/semantics/completor.py @@ -45,7 +45,6 @@ get_entity_types, get_keywords, all_relations, - TRANSFORMS, ) from firepit.timestamp import timefmt @@ -135,8 +134,6 @@ def do_complete( expected_values.append("BY") elif token == "EQUAL": expected_values.append("=") - elif token == "TRANSFORM": - expected_values.extend(TRANSFORMS) elif token == "ATTRIBUTE": # TODO: attribute completion # https://github.com/opencybersecurityalliance/kestrel-lang/issues/79 diff --git a/src/kestrel/syntax/kestrel.lark b/src/kestrel/syntax/kestrel.lark index 6bdb1a14..a8ca41da 100644 --- a/src/kestrel/syntax/kestrel.lark +++ b/src/kestrel/syntax/kestrel.lark @@ -74,13 +74,16 @@ expression: vtrans where_clause? attr_clause? sort_clause? limit_clause? offset_ // not use rule name `transform` since it is a special function in Lark // the function in transformer will mal-function in `merge_transformers()` -vtrans: TRANSFORM "(" VARIABLE ")" +vtrans: transformer "(" VARIABLE ")" | VARIABLE -TRANSFORM: (TIMESTAMPED) +transformer: TIMESTAMPED + | ADDOBSID TIMESTAMPED: "TIMESTAMPED"i +ADDOBSID: "ADDOBSID"i + where_clause: "WHERE"i ecg_pattern attr_clause: "ATTR"i ATTRIBUTES sort_clause: "SORT"i BY ATTRIBUTE (ASC|DESC)? diff --git a/src/kestrel/syntax/parser.py b/src/kestrel/syntax/parser.py index 3163861d..b3196934 100644 --- a/src/kestrel/syntax/parser.py +++ b/src/kestrel/syntax/parser.py @@ -208,10 +208,18 @@ def expression(self, args): return packet def vtrans(self, args): - return { - "input": self._extract_var(args), - "transform": self._assert_and_extract_single("TRANSFORM", args), - } + if len(args) == 1: + return { + "input": self._extract_var(args), + } + else: + return { + "input": self._extract_var(args), + "transformer": args[0], + } + + def transformer(self, args): + return args[0] def where_clause(self, args): pattern = ExtCenteredGraphPattern(args[0]) diff --git a/src/kestrel/syntax/utils.py b/src/kestrel/syntax/utils.py index 8b811067..e19bdf38 100644 --- a/src/kestrel/syntax/utils.py +++ b/src/kestrel/syntax/utils.py @@ -15,8 +15,8 @@ LITERALS = {"CNAME", "LETTER", "DIGIT", "WS", "INT", "WORD", "ESCAPED_STRING", "NUMBER"} AGG_FUNCS = {"MIN", "MAX", "AVG", "SUM", "COUNT", "NUNIQUE"} -TRANSFORMS = {"TIMESTAMPED"} EXPRESSION_OPTIONS = {"WHERE", "ATTR", "SORT", "LIMIT", "OFFSET"} +TRANSFORMS = {"TIMESTAMPED", "ADDOBSID"} def get_keywords():