Merge branch 'imageseries'

This will be a new feature version; imageseries module with ReadGE refactor are functioning and tested.  Note that the v0.3.x version will be maintain for the near future to use with older input format.

Author: Joel Bernier

.gitignore

@@ -10,3 +10,4 @@ hexrd.egg-info
 *.so
 *.*~
 *#
+.#*

conda.recipe/meta.yaml

@@ -9,7 +9,7 @@ source:
 
 build:
   number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }}
-  detect_binary_files_with_prefix: true
+  #  detect_binary_files_with_prefix: true
   osx_is_app: yes
   entry_points:
     - hexrd = hexrd.cli:main
@@ -25,6 +25,7 @@ requirements:
     - python
     - setuptools
   run:
+    - h5py
     - dill
     - matplotlib
     # - nomkl # in case MKL is broken on Linux

docs/source/dev/imageseries.rst

@@ -0,0 +1,17 @@
+imageseries package
+===============
+The *imageseries* package provides a standard API for accessing image-based data sets.  The primary tool in the package is the ImageSeries class. It's interface is analagous to a list of images with  associated image metadata. The number of images is given by the len() function. Properties are defined for image shape (shape), data type (dtype) and metadata (metadata). Individual images are accessed by standard subscripting (e.g. image[i]).
+
+The package contains interfaces for loading (load) and saving (save) imageseries. Images can be loaded in three formats: 'array', 'hdf5' and 'frame-cache'. The 'array' format takes the images from a 3D numpy array. With 'hdf5', images are stored in hdf5 file and accessed on demand. The 'frame-cache' is a list of sparse matrices, useful for thresholded images. An imageseries can be saved in 'hdf5' or 'frame-cache' format.
+
+The imageseries package also contains a  module for modifying the images (process). The process module provides the ProcessedImageSeries class, which takes a given imageseries and produces a new one by modifying the images. It has certain built-in image operations including transposition, flipping, dark subtraction and restriction to a subset.
+
+
+Metadata
+----------------
+
+The metadata property is generically a dictionary. The actual contents depends on the application. For common hexrd applications in which the specimen is rotated while being exposed to x-rays, the metadata has an 'omega' key with an associated value being an nx2 numpy array where n is the number of frames and the two associated values give the omega (rotation) range for that frame.
+
+Reader Refactor
+-------------
+While the imageseries package is in itself indpendent of hexrd, it was used as the basis of a refactoring of the reader classes originally found in the detector module. The main reader class was ReadGE. In the refactored code, the reader classes are now in their own module, image_io, but imported into detector to preserve the interface. The image_io module contains a generic OmegaImageSeries class for working with imageseries having omega metadata. The refactored ReadGE class simply uses the OmegaImageSeries class to provide the same methods as the old class. New code should use the OmegaImageSeries (or the standard ImageSeries) class directly.

docs/source/dev/index.rst

@@ -9,3 +9,4 @@ Contents:
 
    getting_started
    releases
+   imageseries

hexrd/cli/findorientations.py

@@ -92,7 +92,7 @@ def execute(args, parser):
         )
     if os.path.exists(quats_f) and not (args.force or args.clean):
         logger.error(
-            '%s already exists. Change yml file or specify "force"', quats_f
+            '%s already exists. Change yml file or specify "force" or "clean"', quats_f
             )
         sys.exit()
     if not os.path.exists(cfg.working_dir):

hexrd/config/imageseries.py

@@ -2,131 +2,48 @@
 import os
 
 from .config import Config
+from hexrd import imageseries
 
 
+class ImageSeriesConfig(Config):
+
+    def _open(self):
+        self._imser = imageseries.open(self.filename, self.format, **self.args)
 
-class FileConfig(Config):
+    def _meta(self):
+        pass # to be done later
 
 
     @property
-    def stem(self):
-        temp = self._cfg.get('image_series:file:stem')
+    def filename(self):
+        temp = self._cfg.get('image_series:filename')
         if not os.path.isabs(temp):
             temp = os.path.join(self._cfg.working_dir, temp)
         return temp
 
-
-    @property
-    def ids(self):
-        temp = self._cfg.get('image_series:file:ids')
-        return temp if isinstance(temp, list) else [temp]
-
-
-
-class ImagesConfig(Config):
-
-
     @property
-    def start(self):
-        return self._cfg.get('image_series:images:start', default=0)
-
+    def format(self):
+        return self._cfg.get('image_series:format')
 
     @property
-    def step(self):
-        return self._cfg.get('image_series:images:step', default=1)
-
+    def args(self):
+        return self._cfg.get('image_series:args')
 
     @property
-    def stop(self):
-        return self._cfg.get('image_series:images:stop', default=None)
-
+    def omega(self):
+        return OmegaConfig(self._cfg)
 
 
 class OmegaConfig(Config):
 
-
-    @property
-    def start(self):
-        return self._cfg.get('image_series:omega:start')
-
-
     @property
     def step(self):
         return self._cfg.get('image_series:omega:step')
 
+    @property
+    def start(self):
+        return self._cfg.get('image_series:omega:start')
 
     @property
     def stop(self):
         return self._cfg.get('image_series:omega:stop')
-
-
-
-class ImageSeriesConfig(Config):
-
-
-    @property
-    def dark(self):
-        temp = self._cfg.get(
-            'image_series:dark', default=None
-            )
-        if temp is None or os.path.exists(temp):
-            return temp
-        raise IOError(
-            '"image_series:dark": "%s" does not exist' % temp
-            )
-
-
-    @property
-    def file(self):
-        return FileConfig(self._cfg)
-
-
-    @property
-    def files(self):
-        stem = self._cfg.image_series.file.stem
-        res = []
-        missing = []
-        for id in self._cfg.image_series.file.ids:
-            try:
-                id = stem % id
-            except TypeError:
-                # string interpolation failed, join stem and id:
-                id = stem + id
-            temp = glob.glob(id)
-            if temp:
-                res.extend(temp)
-            else:
-                missing.append(id)
-        if missing:
-            raise IOError(
-                'Image files not found: %s' % (', '.join(missing))
-                )
-        return res
-
-
-    @property
-    def flip(self):
-        temp = self._cfg.get('image_series:flip', default=None)
-        if temp is None:
-            return
-        temp = temp.lower()
-        if temp not in ['h', 'v', 'hv', 'vh', 'cw90', 'ccw90']:
-            raise RuntimeError(
-                'image_series:flip setting "%s" is not valid' % temp
-                )
-        return temp
-
-
-    @property
-    def images(self):
-        return ImagesConfig(self._cfg)
-
-
-    @property
-    def omega(self):
-        return OmegaConfig(self._cfg)
-
-
-    @property
-    def n_frames(self):
-        return (self.omega.stop - self.omega.start)/self.omega.step

hexrd/config/root.py

@@ -82,7 +82,7 @@ def multiprocessing(self):
                 if multiproc > ncpus:
                     logger.warning(
                         'Resuested %s processes, %d available',
-                        multiproc, ncpus, ncpus
+                        multiproc, ncpus
                         )
                     res = ncpus
                 else:

hexrd/config/tests/common.py

@@ -9,7 +9,7 @@
 test_data = {
     'existing_path': os.path.abspath('..'),
     'nonexistent_path': 'an_unlikely_name_for_a_directory',
-    'existing_file': __file__,
+    'existing_file': os.path.abspath(__file__),
     'nonexistent_file': 'an_unlikely_name_for_a_file.dat',
     'file_stem': 'test_%%05d.dat',
     'tempdir': tempfile.gettempdir(),

hexrd/config/tests/test_find_orientations.py

@@ -70,7 +70,7 @@ def get_reference_data(cls):
         return reference_data
 
 
-    def test_threshold(self):
+    def test_gvecs(self):
         self.assertFalse(
             self.cfgs[0].find_orientations.extract_measured_g_vectors
             )