Merge pull request #189 from DUNE/reflow-prep

Updates for latest data reflows

Author: mjkramer

data/fsd_flow/light_module_desc-6.0.1.yaml

@@ -23,6 +23,6 @@
                      'scikit-learn>=1.3.0',
                      'h5flow>=0.2.0',
                      'pylandau @ git+https://github.com/cuddandr/pylandau.git#egg=pylandau',
-                     'adc64format @ git+https://github.com/larpix/adc64format.git@v0.1.2#egg=adc64format',
+                     'adc64format @ git+https://github.com/larpix/adc64format.git@v0.1.3#egg=adc64format',
                  ]
                  )

setup.py

@@ -8,7 +8,7 @@
 
 from h5flow.core import resources
 
-from proto_nd_flow.util.array import fill_with_last
+from proto_nd_flow.util.array import fill_with_last, fill_with_next
 
 
 class RawEventBuilder(object):
@@ -147,11 +147,11 @@ def unroll_timestamps(self, packets: np.ndarray) -> np.ndarray:
         ts = (packets['timestamp'].astype('i8') % rollover_ticks) + offsets
 
         # Timestamp packets require special treatment, since their timestamp
-        # field is actually a unix timestamp. For these, we just subtract this
-        # unix timestamp back out, so that their "ts" is the corresponding entry
-        # of "offsets".
+        # field is actually a unix timestamp. For these, we just assign the same
+        # unrolled timestamp as the one in the next non-timestamp packet
         unix_mask = packets['packet_type'] == 4
-        ts[unix_mask] -= packets[unix_mask]['timestamp'].astype('i8')
+        ts[unix_mask] = -1
+        ts = fill_with_next(ts, marker=-1)
 
         return ts
 

src/proto_nd_flow/reco/charge/raw_event_builder.py

@@ -146,6 +146,7 @@ def __init__(self, **params):
 
         # skip to start position
         self.input_file.stream.seek(self.nbytes_runinfo, 0)
+        self.input_file.skip(self.start_position)
 
     def __len__(self):
         return (self.end_position - self.start_position) // (self.batch_size)

src/proto_nd_flow/reco/light/mpd_event_generator.py

@@ -1,16 +1,28 @@
 import numpy as np
 
 
-def fill_with_last(arr: np.ndarray) -> np.ndarray:
+def fill_with_last(arr: np.ndarray, marker=0) -> np.ndarray:
     '''
-        Given an array ARR, copy it and replace all zeros with the last
-        nonzero value. E.g.,
+        Given an array ARR, copy it and replace all elements equal to MARKER
+        (default: 0) with the last value that was not equal to MARKER. E.g.,
         [3, 0, 0, 5, 0, 8, 0, 0, 0] => [3, 3, 3, 5, 5, 8, 8, 8, 8].
         TODO: Replace with a faster implementation. (This one is based on
         the code that calculates unix_ts in raw_event_generator.py.)
     '''
-    groups = np.split(arr, np.argwhere(arr).ravel())
+    groups = np.split(arr, np.argwhere(arr != marker).ravel())
     # SLOW:
     groups = [np.full(len(group), group[0])
               for group in groups if len(group)]
     return np.concatenate(groups, axis=0)
+
+
+def fill_with_next(arr: np.ndarray, marker=0) -> np.ndarray:
+    '''
+        Given an array ARR, copy it and replace all elements equal to MARKER
+        (default: 0) with the next value that is not equal to MARKER. E.g.,
+        [0, 3, 0, 0, 5, 0, 8] => [3, 3, 5, 5, 5, 8, 8].
+        TODO: Replace with a faster implementation.
+    '''
+    a = arr[::-1]
+    a = fill_with_last(a, marker=marker)
+    return a[::-1]

src/proto_nd_flow/util/array.py

@@ -8,7 +8,7 @@ params:
   # configuration parameters
   mc_tracks_dset_name: 'mc_truth/segments'
   buffer_size: 100000
-  nhit_cut: 20
+  nhit_cut: 5
   sync_noise_cut: [100, 11000000] # Stephen says no need to cut > first 10 usec
   sync_noise_cut_enabled: True
   event_builder_class: 'ExtTrigRawEventBuilder' #'SymmetricWindowRawEventBuilder'

yamls/fsd_flow/reco/charge/RawEventGenerator.yaml

@@ -12,9 +12,16 @@ params:
   # output
   hits_dset_name: 'light/sipm_hits' # new dataset to produce
 
+  #threshold: !include yamls/fsd_flow/reco/light/sipm_threshold.yaml
+  threshold: 2
+
   # configuration parameters
   near_samples: 4
   hit_level: 'sipm'
-
-  #threshold: !include yamls/fsd_flow/reco/light/sipm_threshold.yaml
-  threshold: 2
+  mad_factor: 5.0
+  noise_factor: 5.0
+  n_bins_rolled: 5
+  rt_sqrt_factor: 5.0
+  pe_weight: 1.0
+  rising_edge: False
+  local_maxima: True

yamls/fsd_flow/reco/light/SiPMHitFinder.yaml

@@ -12,10 +12,16 @@ params:
   # output
   hits_dset_name: 'light/sum_hits' # new dataset to produce
 
+  #threshold: !include yamls/fsd_flow/reco/light/sipm_threshold.yaml
+  threshold: 4
+
   # configuration parameters
   near_samples: 4
   hit_level: 'sum'
-
-  #threshold: !include yamls/fsd_flow/reco/light/sipm_threshold.yaml
-
-  threshold: 4 
+  mad_factor: 5.0
+  noise_factor: 5.0
+  n_bins_rolled: 5
+  rt_sqrt_factor: 5.0
+  pe_weight: 1.0
+  rising_edge: False
+  local_maxima: True

yamls/fsd_flow/reco/light/SumHitFinder.yaml

@@ -8,7 +8,7 @@ params:
   # crs_geometry_to_module: [0,0,1,0]
   crs_geometry_files: ['data/fsd_flow/multi_tile_layout-3.0.40_fsd_v3.yaml']
   crs_geometry_to_module: [0]
-  lrs_geometry_file: 'data/fsd_flow/light_module_desc-6.0.0.yaml'
+  lrs_geometry_file: 'data/fsd_flow/light_module_desc-6.0.1.yaml'
   beam_direction: 'z'
   drift_direction: 'x'
   network_agnostic: true

yamls/fsd_flow/resources/GeometryData.yaml

@@ -8,7 +8,7 @@ params:
   # crs_geometry_to_module: [0,0,1,0]
   crs_geometry_files: ['data/fsd_flow/multi_tile_layout-3.0.40_NDLArModule_v3.yaml']
   crs_geometry_to_module: [0]
-  lrs_geometry_file: 'data/fsd_flow/light_module_desc-6.0.0.yaml'
+  lrs_geometry_file: 'data/fsd_flow/light_module_desc-6.0.1.yaml'
   beam_direction: 'z'
   drift_direction: 'x'
   network_agnostic: true