gufi_distributed cleanup

add sorting directories found by find(1) randomly
    try to spread consecutive large subtrees across multiple targets

ignore final slurm process stdout

print error messages when jobs fail

Author: calccrypto

scripts/gufi_distributed.py

@@ -63,6 +63,7 @@
 
 import argparse
 import os
+import random
 import shlex
 import subprocess
 import sys
@@ -72,9 +73,10 @@
 
 # how to sort paths found by find(1) at given level
 SORT_DIRS = {
-    'unsorted' : lambda dirs : dirs,
+    'unsorted' : lambda dirs : dirs,         # whatever order find(1) printed in
     'path'     : lambda dirs : sorted(dirs), # pylint: disable=unnecessary-lambda
     'basename' : lambda dirs : sorted(dirs, key = os.path.basename),
+    'random'   : lambda dirs : random.sample(dirs, len(dirs)),
 }
 
 # wait on sbatch, not the actual job
@@ -86,6 +88,7 @@ def run_slurm(args, target, cmd):
     out, _ = proc.communicate()
 
     if proc.returncode != 0:
+        sys.stderr.write('slurm job to {0} failed with error code {1}\n', target, proc.returncode)
         return None
 
     return out.split()[-1].decode()
@@ -99,7 +102,7 @@ def handle_slurm_procs(args, jobids):
                             ['--dependency', 'after:' + after] if len(after) != 0 else [] +
                             ['--wait', '/dev/stdin'],
                             stdin=subprocess.PIPE,
-                            stdout=subprocess.DEVNULL, # Python 3.3
+                            stdout=subprocess.PIPE, # not used
                             shell=True)
 
     # split shebang so shellcheck script doesn't see this
@@ -122,6 +125,8 @@ def handle_ssh_procs(_args, procs):
         proc.wait()
 
         if proc.returncode != 0:
+            sys.stderr.write('ssh to {0} failed with error code {1}\n'.format(proc.args[1], # args[1] only works because there are no arguments between ssh and the target
+                                                                            proc.returncode))
             continue
 
         jobids += [proc.pid]

test/regression/gufi_distributed.expected

@@ -116,6 +116,20 @@ Using existing files
     Process upper directories up to and including level 0 on 127.0.0.1
 Waiting for slurm jobs to complete
 
+# sbatch succeeded but jobs failed (errors are not printed - need to check output files)
+$ gufi_dir2index_distributed --sbatch "sbatch" --gufi_dir2index "false" slurm "hostfile" 1 "prefix" "search2"
+Splitting 4 paths into 5 groups of max size 1
+    Range 0: 1 path on localhost
+        directory directory
+    Range 1: 1 path on 127.0.0.1
+        empty_directory empty_directory
+    Range 2: 1 path on localhost
+        leaf_directory leaf_directory
+    Range 3: 1 path on 127.0.0.1
+        unusual#? directory , unusual#? directory ,
+    Process upper directories up to and including level 0 on 127.0.0.1
+Waiting for slurm jobs to complete
+
 #####################################
 
 #####################################
@@ -231,4 +245,23 @@ Using existing files
     Process upper directories up to and including level 0 on 127.0.0.1
 Waiting for ssh jobs to complete
 
+# ssh succeeded but jobs failed (prints errors)
+$ gufi_dir2index_distributed --ssh "ssh" --gufi_dir2index "false" ssh "hostfile" 1 "prefix" "search2"
+ssh to localhost failed with error code 1
+ssh to 127.0.0.1 failed with error code 1
+ssh to localhost failed with error code 1
+ssh to 127.0.0.1 failed with error code 1
+ssh to 127.0.0.1 failed with error code 1
+Splitting 4 paths into 5 groups of max size 1
+    Range 0: 1 path on localhost
+        directory directory
+    Range 1: 1 path on 127.0.0.1
+        empty_directory empty_directory
+    Range 2: 1 path on localhost
+        leaf_directory leaf_directory
+    Range 3: 1 path on 127.0.0.1
+        unusual#? directory , unusual#? directory ,
+    Process upper directories up to and including level 0 on 127.0.0.1
+Waiting for ssh jobs to complete
+
 #####################################

test/regression/gufi_distributed.sh.in

@@ -213,6 +213,9 @@ run_no_sort "@DIFF@ <(${GUFI_QUERY} -d \" \" -S \"SELECT rpath(sname, sroll) FRO
 echo "# Use existing group files (path_list.4 does not exist)"
 run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --sbatch \"${SBATCH_FAKE}\" --gufi_dir2index \"${GUFI_DIR2INDEX}\" --use-existing-group-files slurm \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${SRCDIR}\" \"${SEARCH2}\" 2>/dev/null"
 
+echo "# sbatch succeeded but jobs failed (errors are not printed - need to check output files)"
+run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --sbatch \"${SBATCH_FAKE}\" --gufi_dir2index \"${FALSE}\" slurm \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${SRCDIR}\" \"${SEARCH2}\""
+
 # cleanup
 cleanup
 echo "#####################################"
@@ -255,6 +258,9 @@ run_no_sort "@DIFF@ <(${GUFI_QUERY} -d \" \" -S \"SELECT rpath(sname, sroll) FRO
 
 echo "# Use existing group files (path_list.4 does not exist)"
 run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --ssh \"${SSH_FAKE}\" --gufi_dir2index \"${GUFI_DIR2INDEX}\" --use-existing-group-files ssh \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${SRCDIR}\" \"${SEARCH2}\" 2>/dev/null | tail -n 9"
+
+echo "# ssh succeeded but jobs failed (prints errors)"
+run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --ssh \"${SSH_FAKE}\" --gufi_dir2index \"${FALSE}\" ssh \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${SRCDIR}\" \"${SEARCH2}\""
 echo "#####################################"
 ) 2>&1 | remove_distributed_output | tee "${OUTPUT}"
 

test/regression/setup.sh.in

@@ -192,6 +192,7 @@ VERIFYTRACEINTREE="@CMAKE_BINARY_DIR@/contrib/verifytraceintree"
 AWK="@AWK@"
 COLUMN="@COLUMN@"
 DIFF="@DIFF@"
+FALSE="$(which false)"
 GREP="@GREP@"
 SED="@SED@"
 STAT="@STAT@"
@@ -276,6 +277,7 @@ replace() {
     s/${AWK//\//\\/}/awk/g;
     s/${COLUMN//\//\\/}/column/g;
     s/${DIFF//\//\\/}/diff/g;
+    s/${FALSE//\//\\/}/false/g;
     s/${GREP//\//\\/}/grep/g;
     s/${SED//\//\\/}/sed/g;
     s/${STAT//\//\\/}/stat/g;