Transformer ADDOBSID added

setup.cfg

@@ -37,7 +37,7 @@ install_requires =
     docker>=5.0.0
     stix-shifter>=5.3.0
     stix-shifter-utils>=5.3.0
-    firepit>=2.3.20
+    firepit>=2.3.21
     typeguard
 tests_require =
     pytest

src/kestrel/codegen/commands.py

@@ -111,10 +111,14 @@ def wrapper(stmt, session):
 @_default_output
 def assign(stmt, session):
     entity_table = session.symtable[stmt["input"]].entity_table
-    transform = stmt.get("transform")
+    transform = stmt.get("transformer")
     if transform:
         if transform.lower() == "timestamped":
             qry = session.store.timestamped(entity_table, run=False)
+        elif transform.lower() == "addobsid":
+            qry = session.store.extract_observeddata_attribute(
+                entity_table, name_of_attribute="id", run=False
+            )
         else:
             qry = Query(entity_table)
     else:
@@ -214,10 +218,14 @@ def info(stmt, session):
 @_debug_logger
 def disp(stmt, session):
     entity_table = session.symtable[stmt["input"]].entity_table
-    transform = stmt.get("transform")
+    transform = stmt.get("transformer")
     if transform and entity_table:
         if transform.lower() == "timestamped":
             qry = session.store.timestamped(entity_table, run=False)
+        elif transform.lower() == "addobsid":
+            qry = session.store.extract_observeddata_attribute(
+                entity_table, name_of_attribute="id", run=False
+            )
         else:
             qry = Query(entity_table)
     else:

src/kestrel/semantics/completor.py

@@ -45,7 +45,6 @@
     get_entity_types,
     get_keywords,
     all_relations,
-    TRANSFORMS,
 )
 from firepit.timestamp import timefmt
 
@@ -135,8 +134,6 @@ def do_complete(
                 expected_values.append("BY")
             elif token == "EQUAL":
                 expected_values.append("=")
-            elif token == "TRANSFORM":
-                expected_values.extend(TRANSFORMS)
             elif token == "ATTRIBUTE":
                 # TODO: attribute completion
                 # https://github.com/opencybersecurityalliance/kestrel-lang/issues/79

src/kestrel/syntax/kestrel.lark

@@ -74,13 +74,16 @@ expression: vtrans where_clause? attr_clause? sort_clause? limit_clause? offset_
 
 // not use rule name `transform` since it is a special function in Lark
 // the function in transformer will mal-function in `merge_transformers()`
-vtrans: TRANSFORM "(" VARIABLE ")"
+vtrans: transformer "(" VARIABLE ")"
       | VARIABLE
 
-TRANSFORM: (TIMESTAMPED)
+transformer: TIMESTAMPED
+           | ADDOBSID
 
 TIMESTAMPED: "TIMESTAMPED"i
 
+ADDOBSID: "ADDOBSID"i
+
 where_clause: "WHERE"i ecg_pattern
 attr_clause: "ATTR"i ATTRIBUTES
 sort_clause: "SORT"i BY ATTRIBUTE (ASC|DESC)?

src/kestrel/syntax/parser.py

@@ -208,10 +208,18 @@ def expression(self, args):
         return packet
 
     def vtrans(self, args):
-        return {
-            "input": self._extract_var(args),
-            "transform": self._assert_and_extract_single("TRANSFORM", args),
-        }
+        if len(args) == 1:
+            return {
+                "input": self._extract_var(args),
+            }
+        else:
+            return {
+                "input": self._extract_var(args),
+                "transformer": args[0],
+            }
+
+    def transformer(self, args):
+        return args[0]
 
     def where_clause(self, args):
         pattern = ExtCenteredGraphPattern(args[0])

src/kestrel/syntax/utils.py

@@ -15,8 +15,8 @@
 
 LITERALS = {"CNAME", "LETTER", "DIGIT", "WS", "INT", "WORD", "ESCAPED_STRING", "NUMBER"}
 AGG_FUNCS = {"MIN", "MAX", "AVG", "SUM", "COUNT", "NUNIQUE"}
-TRANSFORMS = {"TIMESTAMPED"}
 EXPRESSION_OPTIONS = {"WHERE", "ATTR", "SORT", "LIMIT", "OFFSET"}
+TRANSFORMS = {"TIMESTAMPED", "ADDOBSID"}
 
 
 def get_keywords():