add dropoff event to ObjectTracing (#1060)

Author: cfkoehler

src/main/java/emissary/output/filter/AbstractRollableFilter.java

@@ -10,6 +10,8 @@
 import emissary.pool.AgentPool;
 import emissary.roll.RollManager;
 import emissary.roll.Roller;
+import emissary.spi.ObjectTracing;
+import emissary.spi.ObjectTracingService;
 import emissary.util.io.FileNameGenerator;
 
 import org.apache.commons.lang3.StringUtils;
@@ -35,6 +37,7 @@ public abstract class AbstractRollableFilter extends AbstractFilter {
     public static final String MAX_ROLL_FILE_SIZE = "MAX_FILE_SIZE";
     public static final String MAX_OUTPUT_APPENDERS = "MAX_OUTPUT_APPENDERS";
     public static final String ROLL_INTERVAL_UNIT = "ROLL_INTERVAL_UNIT";
+    public static final String ENABLE_OBJECT_TRACE = "ENABLE_OBJECT_TRACE";
 
     protected String defaultOutputPath = "./out";
     protected Path outputPath;
@@ -46,6 +49,7 @@ public abstract class AbstractRollableFilter extends AbstractFilter {
     protected IJournaler rollable;
     protected FileNameGenerator fileNameGenerator;
     protected boolean appendNewLine = true;
+    protected boolean enableObjectTrace = false;
 
     /**
      * Method to convert payload(s) to an output type
@@ -98,6 +102,7 @@ protected void initRollConfig() {
         this.maxOutputAppenders = this.filterConfig.findIntEntry(MAX_OUTPUT_APPENDERS, AgentPool.computePoolSize());
         this.rollInterval = this.filterConfig.findLongEntry(CFG_ROLL_INTERVAL, rollInterval);
         this.rollIntervalUnits = TimeUnit.valueOf(this.filterConfig.findStringEntry(ROLL_INTERVAL_UNIT, rollIntervalUnits.toString()));
+        this.enableObjectTrace = this.filterConfig.findBooleanEntry(ENABLE_OBJECT_TRACE, enableObjectTrace);
     }
 
     /**
@@ -183,6 +188,14 @@ public int filter(final List<IBaseDataObject> payloadList, final Map<String, Obj
             if (code == STATUS_SUCCESS) {
                 ko.commit();
             }
+
+            // Emit dropoff object tracing events
+            if (enableObjectTrace) {
+                for (IBaseDataObject d : payloadList) {
+                    ObjectTracingService.emitLifecycleEvent(d, d.getFilename(), ObjectTracing.Stage.DROP_OFF, true, this.filterName,
+                            String.valueOf(ko.getFinalDestination().getFileName()));
+                }
+            }
         } catch (IOException e) {
             logger.error("IOException during dropoff.", e);
             code = STATUS_FAILURE;

src/main/java/emissary/spi/EmissaryObjectTracingProvider.java

@@ -7,11 +7,21 @@
 public class EmissaryObjectTracingProvider implements ObjectTracing {
 
     @Override
-    public void getObjectTraceFields(IBaseDataObject d, String filename, Stage stage, Map<String, String> fieldMap) {
+    public void getObjectTracePickUpFields(IBaseDataObject d, String filename, Stage stage, Map<String, String> fieldMap) {
         fieldMap.put("inputFileName", filename);
         fieldMap.put("stage", String.valueOf(stage));
     }
 
+    @Override
+    public void getObjectTraceDropOffFields(IBaseDataObject d, String filename, Stage stage, String filterName, String outputFileName,
+            Map<String, String> fieldMap) {
+        fieldMap.put("inputFileName", filename);
+        fieldMap.put("stage", String.valueOf(stage));
+        fieldMap.put("outputFileName", outputFileName);
+        fieldMap.put("uuid", String.valueOf(d.getInternalId()));
+        fieldMap.put("outputType", filterName);
+    }
+
     @Override
     public void mapFieldNames(Map<String, String> fieldMap) {
         // no actions to perform

src/main/java/emissary/spi/ObjectTracing.java

@@ -18,7 +18,20 @@ enum Stage {
      * @param stage The stage
      * @param fieldMap The map of fields we are adding to
      */
-    void getObjectTraceFields(IBaseDataObject d, String filename, ObjectTracing.Stage stage, Map<String, String> fieldMap);
+    void getObjectTracePickUpFields(IBaseDataObject d, String filename, ObjectTracing.Stage stage, Map<String, String> fieldMap);
+
+    /**
+     * With this provider, add the appropriate fields/values to the fieldMap
+     *
+     * @param d The IBDO
+     * @param filename The filename of the object
+     * @param stage The stage
+     * @param filterName The name of the output filter (json, xml, etc.)
+     * @param outputFileName The output filename that is generated
+     * @param fieldMap The map of fields we are adding to
+     */
+    void getObjectTraceDropOffFields(IBaseDataObject d, String filename, Stage stage, String filterName, String outputFileName,
+            Map<String, String> fieldMap);
 
     /**
      * Remaps field names if needed

src/main/java/emissary/spi/ObjectTracingService.java

@@ -22,6 +22,11 @@ public class ObjectTracingService {
     private ObjectTracingService() {}
 
     public static synchronized void emitLifecycleEvent(IBaseDataObject d, String filename, ObjectTracing.Stage stage, boolean useObjectTracing) {
+        emitLifecycleEvent(d, filename, stage, useObjectTracing, null, null);
+    }
+
+    public static synchronized void emitLifecycleEvent(IBaseDataObject d, String filename, ObjectTracing.Stage stage, boolean useObjectTracing,
+            @Nullable String filterName, @Nullable String outputFileName) {
         if (useObjectTracing) {
             if (loader == null) {
                 loader = ServiceLoader.load(ObjectTracing.class);
@@ -30,7 +35,11 @@ public static synchronized void emitLifecycleEvent(IBaseDataObject d, String fil
             // have the appropriate providers add fields
             Map<String, String> jsonFieldMap = new HashMap<>();
             for (ObjectTracing tracing : loader) {
-                tracing.getObjectTraceFields(d, filename, stage, jsonFieldMap);
+                if (stage.equals(ObjectTracing.Stage.PICK_UP)) {
+                    tracing.getObjectTracePickUpFields(d, filename, stage, jsonFieldMap);
+                } else {
+                    tracing.getObjectTraceDropOffFields(d, filename, stage, filterName, outputFileName, jsonFieldMap);
+                }
             }
             // once we have added fields from all providers, perform remapping of field names as appropriate
             for (ObjectTracing tracing : loader) {

src/main/resources/emissary/output/filter/JsonOutputFilter.cfg

@@ -8,3 +8,4 @@ EXTRA_PARAM = "*"
 #DENYLIST_FIELD =
 #DENYLIST_PREFIX =
 #EMIT_PAYLOAD = "false"
+ENABLE_OBJECT_TRACE = "true"