print distributed work runtime

preferrably with monotonic clock

also now printing ssh output
    for index output
    query results will write to files

Author: calccrypto

scripts/gufi_distributed.py

@@ -66,6 +66,7 @@
 import shlex
 import subprocess
 import sys
+import time
 
 import gufi_common
 
@@ -105,7 +106,6 @@ def run_ssh(args, target, cmd):
     # in waiting for the process to complete
     # pylint: disable=consider-using-with
     return subprocess.Popen([args.ssh, target, 'cd', os.getcwd(), '&&'] + [shlex.quote(argv) for argv in cmd],
-                            stdout=subprocess.DEVNULL, # Python 3.3
                             cwd=os.getcwd())
 
 # wait for all jobs to complete
@@ -202,8 +202,25 @@ def schedule_top(args, func):
 
     return DISTRIBUTORS[args.distributor][0](args, target, cmd)
 
+def clock():
+    # assume Python 3+
+    if sys.version_info.minor < 3:
+        return time.time()
+    if sys.version_info.minor < 7:
+        return time.monotonic() # Python 3.3
+    return time.monotonic_ns()  # Python 3.7
+
+def clock_diff(start, end):
+    # assume Python 3+
+    diff = end - start
+    if sys.version_info.minor >= 7:
+        diff /= 1e9
+    return diff
+
 # call this combined function to distribute work
 def distribute_work(args, root, schedule_subtree_func, schedule_top_func):
+    start = clock()
+
     dirs = dirs_at_level(root, args.level)
     group_size, groups = group_dirs(dirs, len(args.hosts[0]), args.sort)
 
@@ -217,7 +234,13 @@ def distribute_work(args, root, schedule_subtree_func, schedule_top_func):
     print('Waiting for {0} jobs to complete'.format(args.distributor))
 
     # wait for actual jobs to return
-    return DISTRIBUTORS[args.distributor][1](args, procs)
+    jobids = DISTRIBUTORS[args.distributor][1](args, procs)
+
+    end = clock()
+
+    print('Jobs completed in {0} seconds'.format(clock_diff(start, end)))
+
+    return jobids
 
 # argparse type for existing directories (i.e. source tree)
 def dir_arg(path):

test/regression/gufi_distributed.sh.in

@@ -161,6 +161,10 @@ trap cleanup_exit EXIT
 
 cleanup
 
+remove_distributed_output() {
+    remove_indexing_time | @GREP@ -v "Creating GUFI\|Total Dirs:\|Total Non-Dirs\|Total Files:\|Jobs completed in"
+}
+
 run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --help"
 run_no_sort "${GUFI_DIR2TRACE_DISTRIBUTED} --help"
 run_no_sort "${GUFI_QUERY_DISTRIBUTED} --help"
@@ -246,7 +250,7 @@ echo "# Diff original index results against the trace files"
 # shellcheck disable=SC2145,SC2153
 run_no_sort "@DIFF@ <(${GUFI_QUERY} -d \" \" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort | @SED@ 's/${SEARCH//\//\\/}\\///g') <(cat ${traces[@]} | @AWK@ -F'${DELIM}' '{ print \$1 }' | sort)"
 echo "#####################################"
-) 2>&1 | tee "${OUTPUT}"
+) 2>&1 | remove_distributed_output | tee "${OUTPUT}"
 
 @DIFF@ @CMAKE_CURRENT_BINARY_DIR@/gufi_distributed.expected "${OUTPUT}"
 rm "${OUTPUT}"