Merge pull request #818 from HEXRD/cli-preload-data

Preload data for CLI fit-grains

Author: psavery

hexrd/fitgrains.py

@@ -386,6 +386,16 @@ def fit_grains(cfg,
     else:
         nproc = min(ncpus, len(grains_table))
         chunksize = max(1, len(grains_table)//ncpus)
+
+        if multiprocessing.get_start_method() == 'fork':
+            # For frame cache, we need to load in all of the data up-front
+            # so it can use fork multiprocessing to share with the other
+            # processes. Otherwise, every process will load in the data on
+            # its own. Accessing one frame in the imageseries is currently
+            # all we need to do to trigger frame caches to load in all the data.
+            for ims in imsd.values():
+                ims[0]
+
         logger.info("\tstarting fit on %d processes with chunksize %d",
                     nproc, chunksize)
         start = timeit.default_timer()

hexrd/imageseries/load/hdf5.py

@@ -67,7 +67,7 @@ def __getitem__(self, key):
         if not isinstance(key, int):
             # FIXME: we do not yet support fancy indexing here.
             # Fully expand the array then apply the fancy indexing.
-            return self[key[0]][*key[1:]]
+            return self[key[0]][tuple(key[1:])]
 
         if self._ndim == 2:
             if key != 0:

hexrd/imageseries/load/imagefiles.py

@@ -48,7 +48,7 @@ def __getitem__(self, key):
         if not isinstance(key, int):
             # FIXME: we do not yet support fancy indexing here.
             # Fully expand the array then apply the fancy indexing.
-            return self[key[0]][*key[1:]]
+            return self[key[0]][tuple(key[1:])]
 
         if self.singleframes:
             frame = None

hexrd/imageseries/load/rawimage.py

@@ -114,7 +114,7 @@ def __getitem__(self, key):
         if not isinstance(key, int):
             # FIXME: we do not yet support fancy indexing here.
             # Fully expand the array then apply the fancy indexing.
-            return self[key[0]][*key[1:]]
+            return self[key[0]][tuple(key[1:])]
 
         count = key * self._frame_bytes + self.skipbytes