File placement improvements

hexrd/cli/find_orientations.py

@@ -1,11 +1,16 @@
 from __future__ import print_function, division, absolute_import
 
 import os
+import logging
+import sys
+
 import numpy as np
 
 from hexrd import constants as const
+from hexrd import config
 from hexrd import instrument
 from hexrd.transforms import xfcapi
+from hexrd.findorientations import find_orientations, write_scored_orientations
 
 
 descr = 'Process rotation image series to find grain orientations'
@@ -50,39 +55,17 @@ def configure_parser(sub_parsers):
     p.set_defaults(func=execute)
 
 
-def write_scored_orientations(results, cfg):
-    # grab working directory from config
-    wdir = cfg.working_dir
-
-    scored_quats_filename = os.path.join(
-        wdir, '_'.join(['scored_orientations', cfg.analysis_id])
-    )
-    np.savez_compressed(
-        scored_quats_filename,
-        **results['scored_orientations']
-    )
-
-
 def write_results(results, cfg):
-    # Write out the data
+    # Write scored orientations.
     write_scored_orientations(results, cfg)
 
-    # grab working directory from config
-    wdir = cfg.working_dir
-
-    if not os.path.exists(cfg.analysis_dir):
-        os.makedirs(cfg.analysis_dir)
-    qbar_filename = os.path.join(
-        wdir,
-        'accepted_orientations_' + cfg.analysis_id + '.dat'
-    )
+    # Write accepted orientations.
+    qbar_filename = str(cfg.find_orientations.accepted_orientations_file)
     np.savetxt(qbar_filename, results['qbar'].T,
                fmt='%.18e', delimiter='\t')
 
-    # ??? do we want to do this by default?
-    gw = instrument.GrainDataWriter(
-        os.path.join(cfg.analysis_dir, 'grains.out')
-    )
+    # Write grains.out.
+    gw = instrument.GrainDataWriter(cfg.find_orientations.grains_file)
     for gid, q in enumerate(results['qbar'].T):
         phi = 2*np.arccos(q[0])
         n = xfcapi.unit_vector(q[1:])
@@ -92,12 +75,6 @@ def write_results(results, cfg):
 
 
 def execute(args, parser):
-    import logging
-    import sys
-
-    from hexrd import config
-    from hexrd.findorientations import find_orientations
-
     # make sure hkls are passed in as a list of ints
     try:
         if args.hkls is not None:
@@ -124,24 +101,20 @@ def execute(args, parser):
     cfg = config.open(args.yml)[0]
 
     # prepare the analysis directory
-    quats_f = os.path.join(
-        cfg.working_dir,
-        'accepted_orientations_%s.dat' % cfg.analysis_id
-        )
-    if os.path.exists(quats_f) and not (args.force or args.clean):
+    quats_f = cfg.find_orientations.accepted_orientations_file
+
+    if (quats_f.exists()) and not (args.force or args.clean):
         logger.error(
             '%s already exists. Change yml file or specify "force" or "clean"',
             quats_f
         )
         sys.exit()
-    if not os.path.exists(cfg.working_dir):
-        os.makedirs(cfg.working_dir)
+
+    # Create analysis directory and any intermediates.
+    cfg.analysis_dir.mkdir(parents=True, exist_ok=True)
 
     # configure logging to file
-    logfile = os.path.join(
-        cfg.working_dir,
-        'find-orientations_%s.log' % cfg.analysis_id
-        )
+    logfile = cfg.find_orientations.logfile
     fh = logging.FileHandler(logfile, mode='w')
     fh.setLevel(log_level)
     fh.setFormatter(

hexrd/cli/fit_grains.py

@@ -8,6 +8,7 @@
 from hexrd import config
 from hexrd import constants as cnst
 from hexrd import instrument
+from hexrd.findorientations import find_orientations
 from hexrd.fitgrains import fit_grains
 from hexrd.transforms import xfcapi
 
@@ -118,21 +119,12 @@ def write_results(
     instr = cfg.instrument.hedm
     nfit = len(fit_results)
 
-    # make output directories
-    if not os.path.exists(cfg.analysis_dir):
-        os.mkdir(cfg.analysis_dir)
-        for det_key in instr.detectors:
-            os.mkdir(os.path.join(cfg.analysis_dir, det_key))
-    else:
-        # make sure panel dirs exist under analysis dir
-        for det_key in instr.detectors:
-            if not os.path.exists(os.path.join(cfg.analysis_dir, det_key)):
-                os.mkdir(os.path.join(cfg.analysis_dir, det_key))
-
+    # Make output directories: analysis directory and a subdirectory for
+    # each panel.
+    for det_key in instr.detectors:
+        (cfg.analysis_dir / det_key).mkdir(parents=True, exist_ok=True)
 
-    gw = instrument.GrainDataWriter(
-        os.path.join(cfg.analysis_dir, grains_filename)
-    )
+    gw = instrument.GrainDataWriter(str(cfg.analysis_dir /grains_filename))
     gd_array = np.zeros((nfit, 21))
     gwa = instrument.GrainDataWriter(array=gd_array)
     for fit_result in fit_results:
@@ -142,7 +134,7 @@ def write_results(
     gwa.close()
 
     gdata = GrainData.from_array(gd_array)
-    gdata.save(os.path.join(cfg.analysis_dir, grains_npz))
+    gdata.save(str(cfg.analysis_dir / grains_npz))
 
 
 def execute(args, parser):
@@ -167,18 +159,14 @@ def execute(args, parser):
     cfg = cfgs[0]
 
     # use path to grains.out to determine if analysis exists
-    # !!! note that the analysis dir pre-pends the working_dir
-    grains_filename = os.path.join(cfg.analysis_dir, 'grains.out')
+    grains_filename = cfg.find_orientations.grains_file
 
     # path to accepted_orientations
-    quats_f = os.path.join(
-        cfg.working_dir,
-        'accepted_orientations_%s.dat' % cfg.analysis_id
-        )
+    quats_f = cfg.find_orientations.accepted_orientations_file
 
     # some conditionals for arg handling
-    have_orientations = os.path.exists(quats_f)
-    existing_analysis = os.path.exists(grains_filename)
+    have_orientations = quats_f.exists()
+    existing_analysis = grains_filename.exists()
     fit_estimate = cfg.fit_grains.estimate
     force_without_estimate = args.force and fit_estimate is None
     new_without_estimate = not existing_analysis and fit_estimate is None
@@ -197,23 +185,33 @@ def execute(args, parser):
         else:
             logger.info("Missing %s, running find-orientations", quats_f)
             logger.removeHandler(ch)
-            from hexrd.findorientations import find_orientations
             results = find_orientations(cfg)
             qbar = results['qbar']
             logger.addHandler(ch)
 
     logger.info('=== begin fit-grains ===')
 
     for cfg in cfgs:
-        # prepare the analysis directory
-        if os.path.exists(cfg.analysis_dir) and not clobber:
+
+        # Check whether an existing analysis exists.
+        grains_filename = cfg.fit_grains.grains_file
+
+        if grains_filename.exists() and not clobber:
             logger.error(
-                'Analysis "%s" at %s already exists.'
-                ' Change yml file or specify "force"',
-                cfg.analysis_name, cfg.analysis_dir
+                'Analysis "%s" already exists. '
+                'Change yml file or specify "force"',
+                cfg.analysis_name
                 )
             sys.exit()
 
+        # Set up analysis directory and output directories.
+        cfg.analysis_dir.mkdir(parents=True, exist_ok=True)
+
+        instr = cfg.instrument.hedm
+        for det_key in instr.detectors:
+            det_dir = cfg.analysis_dir / det_key
+            det_dir.mkdir(exist_ok=True)
+
         # Set HKLs to use.
         if cfg.fit_grains.reset_exclusions:
             excl_p = cfg.fit_grains.exclusion_parameters
@@ -235,26 +233,10 @@ def execute(args, parser):
         )
         logger.info(f'using {using_nhkls} HKLs')
 
-        # make output directories
-        instr = cfg.instrument.hedm
-        if not os.path.exists(cfg.analysis_dir):
-            os.makedirs(cfg.analysis_dir)
-            for det_key in instr.detectors:
-                os.mkdir(os.path.join(cfg.analysis_dir, det_key))
-        else:
-            # make sure panel dirs exist under analysis dir
-            for det_key in instr.detectors:
-                if not os.path.exists(os.path.join(cfg.analysis_dir, det_key)):
-                    os.mkdir(os.path.join(cfg.analysis_dir, det_key))
-
         logger.info('*** begin analysis "%s" ***', cfg.analysis_name)
 
         # configure logging to file for this particular analysis
-        logfile = os.path.join(
-            cfg.working_dir,
-            cfg.analysis_name,
-            'fit-grains.log'
-            )
+        logfile = cfg.fit_grains.logfile
         fh = logging.FileHandler(logfile, mode='w')
         fh.setLevel(log_level)
         ff = logging.Formatter(
@@ -273,17 +255,15 @@ def execute(args, parser):
             pr = profile.Profile()
             pr.enable()
 
-        grains_filename = os.path.join(
-            cfg.analysis_dir, 'grains.out'
-        )
 
         # some conditionals for arg handling
-        existing_analysis = os.path.exists(grains_filename)
+        existing_analysis = grains_filename.exists()
         fit_estimate = cfg.fit_grains.estimate
         new_with_estimate = not existing_analysis and fit_estimate is not None
         new_without_estimate = not existing_analysis and fit_estimate is None
         force_with_estimate = args.force and fit_estimate is not None
         force_without_estimate = args.force and fit_estimate is None
+        #
         # ------- handle args
         # - 'clean' indicates ignoring any estimate specified and starting with
         #   the 'accepted_orientations' file.  Will run find-orientations if
@@ -292,6 +272,7 @@ def execute(args, parser):
         #   option "fit_grains:estimate" is None, will use results from
         #   find-orientations.  If 'accepted_orientations' does not exists,
         #   then it runs find-orientations.
+        #
         if args.clean or force_without_estimate or new_without_estimate:
             # need accepted orientations from indexing in this case
             if args.clean:
@@ -304,6 +285,8 @@ def execute(args, parser):
                     + "using default"
                 )
             try:
+                # Write the accepted orientations (in `qbar`) to the
+                # grains.out file
                 gw = instrument.GrainDataWriter(grains_filename)
                 for i_g, q in enumerate(qbar.T):
                     phi = 2*np.arccos(q[0])
@@ -316,7 +299,7 @@ def execute(args, parser):
             except(IOError):
                 raise(RuntimeError,
                       "indexing results '%s' not found!"
-                      % 'accepted_orientations_' + cfg.analysis_id + '.dat')
+                      % str(grains_filename))
         elif force_with_estimate or new_with_estimate:
             grains_filename = fit_estimate
             logger.info("using initial estimate '%s'", fit_estimate)

hexrd/config/config.py

@@ -8,12 +8,23 @@
 logger = logging.getLogger('hexrd.config')
 
 class Config(object):
+    """Access a level of the YAML configuration file
 
+    PARAMETERS
+    ----------
+    cfg: Config instance or a (pyyaml) YAMLObject
+       config representings a level of the YAML input
+    """
     _dirty = False
 
     def __init__(self, cfg):
         self._cfg = cfg
 
+    @property
+    def parent(self):
+        """Parent Config file or None for root (top) level"""
+        return self._cfg if isinstance(self._cfg, Config) else None
+
     @property
     def dirty(self):
         return self._dirty