change -D lhs rhs to exact match

[lhs.*, rhs.*] -> [lhs.*, rhs]

Remove duplicate tree walk where there are paths that have matching prefixes:
    a, aa, aaa, aaaa

    Previously, using 2 nodes:
        Groups: [a.*, aa.*] and [aaa.*, aaaa.*]
        Work done:
            Node 1: a, aa, aaa, aaaa
            Node 2: aaa, aaaa

Updated gufi_distributed text to better reflect what is happening

This still has the issue of potentially processing a disproportionate
number of directories with similar names on the same node because they
fall within the same group
    Instead of using -D, an alternative method may be to pass in the
    actual source paths at level L as input arguments. However, there
    is a limit to shell command size, and there is no guarantee that
    all paths at level L within one group will fit in one command,
    meaning the requested node count cannot be honored.

Author: calccrypto

include/str.h

@@ -124,8 +124,8 @@ typedef struct str_range {
 } str_range_t;
 
 /*
- * [lhs.*, rhs.*]
- * prefix match
+ * [lhs.*, rhs]
+ * exact match
  */
 int str_range_cmp(const str_range_t *range, const refstr_t *str);
 /* ****************************************************** */

scripts/gufi_distributed.py

@@ -97,25 +97,22 @@ def dirs_at_level(root, level):
     return sorted([path.decode() for path in dirs.split(b'\x00') if len(path) > 0])
 
 # step 2
-# split directories into groups for processing
+# split directories into groups of unique basenames for processing
 def group_dirs(dirs, splits):
-    count = len(dirs)
+    basenames = list(set(os.path.basename(path) for path in dirs))
+    count = len(basenames)
     group_size = count // splits + int(bool(count % splits))
-    ordered = sorted(dirs, key=os.path.basename)
-    return group_size, [ordered[i: i + group_size] for i in range(0, count, group_size)]
-
-def dir_plural(count):
-    return 'directories' if count > 1 else 'directory'
+    ordered = sorted(basenames)
+    return count, group_size, [ordered[i: i + group_size] for i in range(0, count, group_size)]
 
 # step 3
 # get only the first and last paths in each group
 # print debug messages
 # run function to schedule jobs if it exists
-def schedule_subtrees(dir_count, splits, group_size, groups, schedule_subtree):
-    print('Splitting {0} {1} into {2} chunks of max size {3}'.format(dir_count,
-                                                                     dir_plural(dir_count),
-                                                                     splits,
-                                                                     group_size))
+def schedule_subtrees(unique_basenames, splits, group_size, groups, schedule_subtree):
+    print('Splitting {0} unique basenames into {1} groups of max size {2}'.format(unique_basenames,
+                                                                                  splits,
+                                                                                  group_size))
 
     jobids = []
     for i, group in enumerate(groups):
@@ -124,7 +121,7 @@ def schedule_subtrees(dir_count, splits, group_size, groups, schedule_subtree):
         if count == 0:
             break
 
-        print('    Range {0}: {1} {2}'.format(i, count, dir_plural(count)))
+        print('    Range {0}: {1} basename{2}'.format(i, count, 's' if count != 1 else ''))
         print('        {0} {1}'.format(group[0], group[-1]))
 
         if schedule_subtree is not None:
@@ -149,8 +146,8 @@ def schedule_top(func, jobids):
 # call this combined function to distribute work
 def distribute_work(root, level, nodes, schedule_subtree_func, schedule_top_func):
     dirs = dirs_at_level(root, level)
-    group_size, groups = group_dirs(dirs, nodes)
-    jobids = schedule_subtrees(len(dirs), nodes, group_size, groups, schedule_subtree_func)
+    unique_basenames, group_size, groups = group_dirs(dirs, nodes)
+    jobids = schedule_subtrees(unique_basenames, nodes, group_size, groups, schedule_subtree_func)
     jobids += [schedule_top(schedule_top_func, jobids).decode()]
     return jobids
 

src/str.c

@@ -92,32 +92,23 @@ void str_free_existing(str_t *str) {
 
 int str_cmp(const str_t *lhs, const str_t *rhs) {
     const size_t len = ((lhs->len > rhs->len)?lhs:rhs)->len;
-    return strncmp(lhs->data, rhs->data, len);
+    return strncmp(lhs->data, rhs->data, len + 1);
 }
 
 int refstr_cmp(const refstr_t *lhs, const refstr_t *rhs) {
     const size_t len = ((lhs->len > rhs->len)?lhs:rhs)->len;
-    return strncmp(lhs->data, rhs->data, len);
-}
-
-/*
- * Compare string prefixes, so that
- * "hello" and "hello world" returns 0
- * but "hello world" and "hello" returns (int) ' '
- */
-static int refstr_cmp_prefix(const refstr_t *prefix, const refstr_t *str) {
-    return strncmp(prefix->data, str->data, prefix->len);
+    return strncmp(lhs->data, rhs->data, len + 1);
 }
 
 int str_range_cmp(const str_range_t *range, const refstr_t *str) {
     /* less than lhs */
-    const int lhc = refstr_cmp_prefix(&range->lhs, str);
+    const int lhc = refstr_cmp(&range->lhs, str);
     if (lhc > 0) {
         return -1;
     }
 
     /* greater than rhs */
-    const int rhc = refstr_cmp_prefix(&range->rhs, str);
+    const int rhc = refstr_cmp(&range->rhs, str);
     if (rhc < 0) {
         return +1;
     }

test/regression/gufi_distributed.expected

@@ -4,28 +4,28 @@ $ gufi_dir2index_distributed --sbatch "sbatch" --gufi_dir2index "gufi_dir2index"
 "search2" Already exists!
 "search2" Already exists!
 "search2" Already exists!
-Splitting 4 directories into 5 chunks of max size 1
-    Range 0: 1 directory
-        prefix/directory prefix/directory
-    Range 1: 1 directory
-        prefix/empty_directory prefix/empty_directory
-    Range 2: 1 directory
-        prefix/leaf_directory prefix/leaf_directory
-    Range 3: 1 directory
-        prefix/unusual#? directory , prefix/unusual#? directory ,
+Splitting 4 unique basenames into 5 groups of max size 1
+    Range 0: 1 basename
+        directory directory
+    Range 1: 1 basename
+        empty_directory empty_directory
+    Range 2: 1 basename
+        leaf_directory leaf_directory
+    Range 3: 1 basename
+        unusual#? directory , unusual#? directory ,
     Index upper directories up to and including level 0
 
 # Query Index
 $ gufi_query_distributed --sbatch "sbatch" --gufi_query "gufi_query" --threads 2 1 5 "prefix2" -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;"
-Splitting 4 directories into 5 chunks of max size 1
-    Range 0: 1 directory
-        prefix2/directory prefix2/directory
-    Range 1: 1 directory
-        prefix2/empty_directory prefix2/empty_directory
-    Range 2: 1 directory
-        prefix2/leaf_directory prefix2/leaf_directory
-    Range 3: 1 directory
-        prefix2/unusual#? directory , prefix2/unusual#? directory ,
+Splitting 4 unique basenames into 5 groups of max size 1
+    Range 0: 1 basename
+        directory directory
+    Range 1: 1 basename
+        empty_directory empty_directory
+    Range 2: 1 basename
+        leaf_directory leaf_directory
+    Range 3: 1 basename
+        unusual#? directory , unusual#? directory ,
     Query upper directories up to and including level 0
 cat the following slurm job output files to get complete results:
     0
@@ -66,15 +66,15 @@ $ diff <(gufi_query -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT r
 
 # Convert source tree to trace files
 $ gufi_dir2trace_distributed --sbatch "sbatch" --gufi_dir2trace "gufi_dir2trace" -d "|" 1 5 "prefix" "traces"
-Splitting 4 directories into 5 chunks of max size 1
-    Range 0: 1 directory
-        prefix/directory prefix/directory
-    Range 1: 1 directory
-        prefix/empty_directory prefix/empty_directory
-    Range 2: 1 directory
-        prefix/leaf_directory prefix/leaf_directory
-    Range 3: 1 directory
-        prefix/unusual#? directory , prefix/unusual#? directory ,
+Splitting 4 unique basenames into 5 groups of max size 1
+    Range 0: 1 basename
+        directory directory
+    Range 1: 1 basename
+        empty_directory empty_directory
+    Range 2: 1 basename
+        leaf_directory leaf_directory
+    Range 3: 1 basename
+        unusual#? directory , unusual#? directory ,
     Index upper directories up to and including level 0
 Index can now be created from "traces.*"
 

test/regression/gufi_query.expected

@@ -254,8 +254,8 @@ prefix/leaf_directory/leaf_file2
 prefix/unusual#? directory ,
 prefix/unusual#? directory ,/unusual, name?#
 
-# limit tree traversal to all directories under [a.*, e.*] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/
-$ gufi_query -d " " -n 2 -y 1 -D a e -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix"
+# limit tree traversal to all directories under [a, f] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/
+$ gufi_query -d " " -n 2 -y 1 -D a f -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix"
 prefix/directory
 prefix/directory/executable
 prefix/directory/readonly

test/regression/gufi_query.sh.in

@@ -127,8 +127,8 @@ run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -S \"SELECT rpath(sname, sroll),
 echo "# limit tree traversal to directories at level 1 (missing prefix/ and prefix/directory/subdirectory/)"
 run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -y 1 -z 1 -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\""
 
-echo "# limit tree traversal to all directories under [a.*, e.*] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/"
-run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -y 1 -D a e -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\""
+echo "# limit tree traversal to all directories under [a, f] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/"
+run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -y 1 -D a f -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\""
 
 echo "# Output TLV columns (no aggregation)"
 run_no_sort "${GUFI_QUERY} -u -n ${THREADS} -E \"SELECT name, size FROM vrpentries WHERE name == '.hidden';\" \"${INDEXROOT}\" | ${HEXLIFY}"

test/unit/googletest/str.cpp

@@ -109,35 +109,47 @@ TEST(str, cmp) {
 }
 
 TEST(str, range) {
+    // before lhs
     refstr_t a;
     a.data = "a";
     a.len  = 1;
 
+    // before lhs
     refstr_t ba;
     ba.data = "ba";
     ba.len  = 2;
 
+    // lhs == bb
     refstr_t bb;
     bb.data = "bb";
     bb.len  = 2;
 
+    // lhs < cc < rhs
     refstr_t cc;
     cc.data = "cc";
     cc.len  = 2;
 
+    // rhs == 'dd'
     refstr_t dd;
     dd.data = "dd";
     dd.len  = 2;
 
+    // after rhs
+    refstr_t ddd;
+    ddd.data = "ddd";
+    ddd.len  = 3;
+
+    // after rhs
     refstr_t de;
     de.data = "de";
     de.len  = 2;
 
+    // after rhs
     refstr_t e;
     e.data = "e";
     e.len  = 1;
 
-    // [bb.*, dd.*]
+    // [bb, dd]
     str_range_t range;
     range.lhs = bb;
     range.rhs = dd;
@@ -147,6 +159,7 @@ TEST(str, range) {
     EXPECT_EQ(str_range_cmp(&range, &bb),  0);
     EXPECT_EQ(str_range_cmp(&range, &cc),  0);
     EXPECT_EQ(str_range_cmp(&range, &dd),  0);
+    EXPECT_EQ(str_range_cmp(&range, &ddd), 1);
     EXPECT_EQ(str_range_cmp(&range, &de),  1);
     EXPECT_EQ(str_range_cmp(&range, &e),   1);
 }