fit-grains CLI implementation

Author: joelvbernier

hexrd/cli/findorientations.py

@@ -1,7 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 
-descr = 'Process diffraction data to find grain orientations'
+descr = 'Process rotation image series to find grain orientations'
 example = """
 examples:
     hexrd find-orientations configuration.yml
@@ -79,29 +79,24 @@ def execute(args, parser):
     # load the configuration settings
     cfg = config.open(args.yml)[0]
 
-    # ...make this an attribute in cfg?
-    analysis_id = '%s_%s' %(
-        cfg.analysis_name.strip().replace(' ', '-'),
-        cfg.material.active.strip().replace(' ', '-'),
-        )
-    
     # prepare the analysis directory
     quats_f = os.path.join(
         cfg.working_dir,
-        'accepted_orientations_%s.dat' %analysis_id
+        'accepted_orientations_%s.dat' % cfg.analysis_id
         )
     if os.path.exists(quats_f) and not (args.force or args.clean):
         logger.error(
-            '%s already exists. Change yml file or specify "force" or "clean"', quats_f
-            )
+            '%s already exists. Change yml file or specify "force" or "clean"',
+            quats_f
+        )
         sys.exit()
     if not os.path.exists(cfg.working_dir):
         os.makedirs(cfg.working_dir)
 
     # configure logging to file
     logfile = os.path.join(
         cfg.working_dir,
-        'find-orientations_%s.log' %analysis_id
+        'find-orientations_%s.log' % cfg.analysis_id
         )
     fh = logging.FileHandler(logfile, mode='w')
     fh.setLevel(log_level)
@@ -120,7 +115,12 @@ def execute(args, parser):
         pr.enable()
 
     # process the data
-    find_orientations(cfg, hkls=args.hkls, clean=args.clean, profile=args.profile)
+    find_orientations(
+        cfg,
+        hkls=args.hkls,
+        clean=args.clean,
+        profile=args.profile
+    )
 
     if args.profile:
         pr.disable()

hexrd/cli/fitgrains.py

@@ -24,11 +24,11 @@ def configure_parser(sub_parsers):
         )
     p.add_argument(
         '-c', '--clean', action='store_true',
-        help='overwrites existing analysis, including frame cache'
+        help='overwrites existing analysis, uses initial orientations'
         )
     p.add_argument(
         '-f', '--force', action='store_true',
-        help='overwrites existing analysis, exlcuding frame cache'
+        help='overwrites existing analysis'
         )
     p.add_argument(
         '-p', '--profile', action='store_true',
@@ -62,17 +62,11 @@ def execute(args, parser):
     cf = logging.Formatter('%(asctime)s - %(message)s', '%y-%m-%d %H:%M:%S')
     ch.setFormatter(cf)
     logger.addHandler(ch)
-
-    # ...make this an attribute in cfg?
-    analysis_id = '%s_%s' %(
-        cfgs[0].analysis_name.strip().replace(' ', '-'),
-        cfgs[0].material.active.strip().replace(' ', '-'),
-        )
 
     # if find-orientations has not already been run, do so:
     quats_f = os.path.join(
         cfgs[0].working_dir, 
-        'accepted_orientations_%s.dat' %analysis_id
+        'accepted_orientations_%s.dat' % cfgs[0].analysis_id
         )
     if not os.path.exists(quats_f):
         logger.info("Missing %s, running find-orientations", quats_f)

hexrd/config/fitgrains.py

@@ -116,7 +116,7 @@ def fit_only(self):
     def tth_max(self):
         key = 'fit_grains:tth_max'
         temp = self._cfg.get(key, True)
-        if temp in (True, False):
+        if isinstance(temp, bool):
             return temp
         if isinstance(temp, (int, float)):
             if temp > 0:

hexrd/findorientations.py

@@ -8,6 +8,8 @@
 import numpy as np
 # np.seterr(over='ignore', invalid='ignore')
 
+import tqdm
+
 import scipy.cluster as cluster
 from scipy import ndimage
 from skimage.feature import blob_dog, blob_log
@@ -62,7 +64,7 @@ def generate_orientation_fibers(cfg, eta_ome):
     #       default values for each case?  They must be specified as of now.
     method = next(method_dict.iterkeys())
     method_kwargs = method_dict[method]
-    logger.info('using "%s" method for fiber generation'
+    logger.info('\tusing "%s" method for fiber generation'
                 % method)
 
     # seed_hkl_ids must be consistent with this...
@@ -163,24 +165,44 @@ def generate_orientation_fibers(cfg, eta_ome):
     if ncpus > 1:
         # multiple process version
         # ???: Need a chunksize in map?
+        chunksize = max(1, len(input_p)//(10*ncpus))
         pool = mp.Pool(ncpus, discretefiber_init, (params, ))
-        qfib = pool.map(discretefiber_reduced, input_p)
+        qfib = pool.map(
+            discretefiber_reduced, input_p,
+            chunksize=chunksize
+        )
+        '''
+        # This is an experiment...
+        ntotal= 10*ncpus + np.remainder(len(input_p), 10*ncpus) > 0
+        for _ in tqdm.tqdm(
+                pool.imap_unordered(
+                    discretefiber_reduced, input_p, chunksize=chunksize
+                ), total=ntotal
+            ):
+            pass
+        print(_.shape)
+        '''
         pool.close()
+        pool.join()
     else:
         # single process version.
-        global paramMP
         discretefiber_init(params)  # sets paramMP
         qfib = map(discretefiber_reduced, input_p)
-        paramMP = None  # clear paramMP
+        discretefiber_cleanup()
     elapsed = (timeit.default_timer() - start)
-    logger.info("fiber generation took %.3f seconds", elapsed)
+    logger.info("\tfiber generation took %.3f seconds", elapsed)
     return np.hstack(qfib)
 
 
 def discretefiber_init(params):
     global paramMP
     paramMP = params
 
+    
+def discretefiber_cleanup():
+    global paramMP
+    del paramMP
+    
 
 def discretefiber_reduced(params_in):
     """
@@ -587,6 +609,7 @@ def find_orientations(cfg,
         # handle search space
         if cfg.find_orientations.use_quaternion_grid is None:
             # doing seeded search
+            logger.info("Will perform seeded search")
             logger.info(
                 "\tgenerating search quaternion list using %d processes",
                 ncpus
@@ -653,7 +676,7 @@ def find_orientations(cfg,
         # do map-based indexing
         start = timeit.default_timer()
 
-        logger.info(" will test %d quaternions using %d processes",
+        logger.info("will test %d quaternions using %d processes",
                     qfib.shape[1], ncpus)
 
         completeness = indexer.paintGrid(
@@ -740,8 +763,6 @@ def find_orientations(cfg,
 
     logger.info("\tmean reflections per grain: %d", mean_rpg)
     logger.info("\tneighborhood size: %d", min_samples)
-    logger.info("\tFeeding %d orientations above %.1f%% to clustering",
-                sum(completeness > compl_thresh), compl_thresh)
 
     qbar, cl = run_cluster(
         completeness, qfib, plane_data.getQSym(), cfg,