change -D lhs rhs to exact match

calccrypto · calccrypto · commit 708a63313642 · 2025-07-18T22:43:48.000Z
[lhs.*, rhs.*] -&gt; [lhs.*, rhs]

Remove duplicate tree walk where there are paths that have matching prefixes:
    a, aa, aaa, aaaa

    Previously, using 2 nodes:
        Groups: [a.*, aa.*] and [aaa.*, aaaa.*]
        Work done:
            Node 1: a, aa, aaa, aaaa
            Node 2: aaa, aaaa

Updated gufi_distributed text to better reflect what is happening

This still has the issue of potentially processing many directories on
the same node if many directories fall within the same group
    Instead of using -D, an alternative method may be to pass in the
    actual source paths at level L as input arguments. However, there
    is a limit to shell command size, and there is no guarantee that
    all paths at level L within one group will fit in one command,
    meaning the requested node count cannot be honored.
diff --git a/include/str.h b/include/str.h
@@ -124,8 +124,8 @@ typedef struct str_range {
 } str_range_t;
 
 /*
- * [lhs.*, rhs.*]
- * prefix match
+ * [lhs.*, rhs]
+ * exact match
  */
 int str_range_cmp(const str_range_t *range, const refstr_t *str);
 /* ****************************************************** */
diff --git a/scripts/gufi_distributed.py b/scripts/gufi_distributed.py
@@ -97,25 +97,22 @@ def dirs_at_level(root, level):
     return sorted([path.decode() for path in dirs.split(b'\x00') if len(path) > 0])
 
 # step 2
-# split directories into groups for processing
+# split dirs into groups of unique basenames for processing
 def group_dirs(dirs, splits):
-    count = len(dirs)
+    basenames = list(set(os.path.basename(path) for path in dirs))
+    count = len(basenames)
     group_size = count // splits + int(bool(count % splits))
-    ordered = sorted(dirs, key=os.path.basename)
-    return group_size, [ordered[i: i + group_size] for i in range(0, count, group_size)]
-
-def dir_plural(count):
-    return 'directories' if count > 1 else 'directory'
+    ordered = sorted(basenames)
+    return count, group_size, [ordered[i: i + group_size] for i in range(0, count, group_size)]
 
 # step 3
 # get only the first and last paths in each group
 # print debug messages
 # run function to schedule jobs if it exists
-def schedule_subtrees(dir_count, splits, group_size, groups, schedule_subtree):
-    print('Splitting {0} {1} into {2} chunks of max size {3}'.format(dir_count,
-                                                                     dir_plural(dir_count),
-                                                                     splits,
-                                                                     group_size))
+def schedule_subtrees(unique_basenames, splits, group_size, groups, schedule_subtree):
+    print('Splitting {0} unique basenames into {1} groups of max size {2}'.format(unique_basenames,
+                                                                                  splits,
+                                                                                  group_size))
 
     jobids = []
     for i, group in enumerate(groups):
@@ -124,7 +121,7 @@ def schedule_subtrees(dir_count, splits, group_size, groups, schedule_subtree):
         if count == 0:
             break
 
-        print('    Range {0}: {1} {2}'.format(i, count, dir_plural(count)))
+        print('    Range {0}: {1} basename{2}'.format(i, count, 's' if count != 1 else ''))
         print('        {0} {1}'.format(group[0], group[-1]))
 
         if schedule_subtree is not None:
@@ -149,8 +146,8 @@ def schedule_top(func, jobids):
 # call this combined function to distribute work
 def distribute_work(root, level, nodes, schedule_subtree_func, schedule_top_func):
     dirs = dirs_at_level(root, level)
-    group_size, groups = group_dirs(dirs, nodes)
-    jobids = schedule_subtrees(len(dirs), nodes, group_size, groups, schedule_subtree_func)
+    unique_basenames, group_size, groups = group_dirs(dirs, nodes)
+    jobids = schedule_subtrees(unique_basenames, nodes, group_size, groups, schedule_subtree_func)
     jobids += [schedule_top(schedule_top_func, jobids).decode()]
     return jobids
 
diff --git a/src/str.c b/src/str.c
@@ -92,32 +92,23 @@ void str_free_existing(str_t *str) {
 
 int str_cmp(const str_t *lhs, const str_t *rhs) {
     const size_t len = ((lhs->len > rhs->len)?lhs:rhs)->len;
-    return strncmp(lhs->data, rhs->data, len);
+    return strncmp(lhs->data, rhs->data, len + 1);
 }
 
 int refstr_cmp(const refstr_t *lhs, const refstr_t *rhs) {
     const size_t len = ((lhs->len > rhs->len)?lhs:rhs)->len;
-    return strncmp(lhs->data, rhs->data, len);
-}
-
-/*
- * Compare string prefixes, so that
- * "hello" and "hello world" returns 0
- * but "hello world" and "hello" returns (int) ' '
- */
-static int refstr_cmp_prefix(const refstr_t *prefix, const refstr_t *str) {
-    return strncmp(prefix->data, str->data, prefix->len);
+    return strncmp(lhs->data, rhs->data, len + 1);
 }
 
 int str_range_cmp(const str_range_t *range, const refstr_t *str) {
     /* less than lhs */
-    const int lhc = refstr_cmp_prefix(&range->lhs, str);
+    const int lhc = refstr_cmp(&range->lhs, str);
     if (lhc > 0) {
         return -1;
     }
 
     /* greater than rhs */
-    const int rhc = refstr_cmp_prefix(&range->rhs, str);
+    const int rhc = refstr_cmp(&range->rhs, str);
     if (rhc < 0) {
         return +1;
     }
diff --git a/test/regression/gufi_distributed.expected b/test/regression/gufi_distributed.expected
@@ -4,28 +4,28 @@ $ gufi_dir2index_distributed --sbatch "sbatch" --gufi_dir2index "gufi_dir2index"
 "search2" Already exists!
 "search2" Already exists!
 "search2" Already exists!
-Splitting 4 directories into 5 chunks of max size 1
-    Range 0: 1 directory
-        prefix/directory prefix/directory
-    Range 1: 1 directory
-        prefix/empty_directory prefix/empty_directory
-    Range 2: 1 directory
-        prefix/leaf_directory prefix/leaf_directory
-    Range 3: 1 directory
-        prefix/unusual#? directory , prefix/unusual#? directory ,
+Splitting 4 unique basenames into 5 groups of max size 1
+    Range 0: 1 basename
+        directory directory
+    Range 1: 1 basename
+        empty_directory empty_directory
+    Range 2: 1 basename
+        leaf_directory leaf_directory
+    Range 3: 1 basename
+        unusual#? directory , unusual#? directory ,
     Index upper directories up to and including level 0
 
 # Query Index
 $ gufi_query_distributed --sbatch "sbatch" --gufi_query "gufi_query" --threads 2 1 5 "prefix2" -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;"
-Splitting 4 directories into 5 chunks of max size 1
-    Range 0: 1 directory
-        prefix2/directory prefix2/directory
-    Range 1: 1 directory
-        prefix2/empty_directory prefix2/empty_directory
-    Range 2: 1 directory
-        prefix2/leaf_directory prefix2/leaf_directory
-    Range 3: 1 directory
-        prefix2/unusual#? directory , prefix2/unusual#? directory ,
+Splitting 4 unique basenames into 5 groups of max size 1
+    Range 0: 1 basename
+        directory directory
+    Range 1: 1 basename
+        empty_directory empty_directory
+    Range 2: 1 basename
+        leaf_directory leaf_directory
+    Range 3: 1 basename
+        unusual#? directory , unusual#? directory ,
     Query upper directories up to and including level 0
 cat the following slurm job output files to get complete results:
     0
@@ -66,15 +66,15 @@ $ diff <(gufi_query -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT r
 
 # Convert source tree to trace files
 $ gufi_dir2trace_distributed --sbatch "sbatch" --gufi_dir2trace "gufi_dir2trace" -d "|" 1 5 "prefix" "traces"
-Splitting 4 directories into 5 chunks of max size 1
-    Range 0: 1 directory
-        prefix/directory prefix/directory
-    Range 1: 1 directory
-        prefix/empty_directory prefix/empty_directory
-    Range 2: 1 directory
-        prefix/leaf_directory prefix/leaf_directory
-    Range 3: 1 directory
-        prefix/unusual#? directory , prefix/unusual#? directory ,
+Splitting 4 unique basenames into 5 groups of max size 1
+    Range 0: 1 basename
+        directory directory
+    Range 1: 1 basename
+        empty_directory empty_directory
+    Range 2: 1 basename
+        leaf_directory leaf_directory
+    Range 3: 1 basename
+        unusual#? directory , unusual#? directory ,
     Index upper directories up to and including level 0
 Index can now be created from "traces.*"
 
diff --git a/test/regression/gufi_query.expected b/test/regression/gufi_query.expected
@@ -254,8 +254,8 @@ prefix/leaf_directory/leaf_file2
 prefix/unusual#? directory ,
 prefix/unusual#? directory ,/unusual, name?#
 
-# limit tree traversal to all directories under [a.*, e.*] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/
-$ gufi_query -d " " -n 2 -y 1 -D a e -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix"
+# limit tree traversal to all directories under [a, f] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/
+$ gufi_query -d " " -n 2 -y 1 -D a f -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix"
 prefix/directory
 prefix/directory/executable
 prefix/directory/readonly
diff --git a/test/regression/gufi_query.sh.in b/test/regression/gufi_query.sh.in
@@ -127,8 +127,8 @@ run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -S \"SELECT rpath(sname, sroll),
 echo "# limit tree traversal to directories at level 1 (missing prefix/ and prefix/directory/subdirectory/)"
 run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -y 1 -z 1 -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\""
 
-echo "# limit tree traversal to all directories under [a.*, e.*] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/"
-run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -y 1 -D a e -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\""
+echo "# limit tree traversal to all directories under [a, f] at level 1 (missing prefix/, prefix/leaf_directory/, and prefix/unusual#? directory ,/"
+run_sort "${GUFI_QUERY} -d \" \" -n ${THREADS} -y 1 -D a f -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\""
 
 echo "# Output TLV columns (no aggregation)"
 run_no_sort "${GUFI_QUERY} -u -n ${THREADS} -E \"SELECT name, size FROM vrpentries WHERE name == '.hidden';\" \"${INDEXROOT}\" | ${HEXLIFY}"
diff --git a/test/unit/googletest/str.cpp b/test/unit/googletest/str.cpp
@@ -109,35 +109,47 @@ TEST(str, cmp) {
 }
 
 TEST(str, range) {
+    // before lhs
     refstr_t a;
     a.data = "a";
     a.len  = 1;
 
+    // before lhs
     refstr_t ba;
     ba.data = "ba";
     ba.len  = 2;
 
+    // lhs == bb
     refstr_t bb;
     bb.data = "bb";
     bb.len  = 2;
 
+    // lhs < cc < rhs
     refstr_t cc;
     cc.data = "cc";
     cc.len  = 2;
 
+    // rhs == 'dd'
     refstr_t dd;
     dd.data = "dd";
     dd.len  = 2;
 
+    // after rhs
+    refstr_t ddd;
+    ddd.data = "ddd";
+    ddd.len  = 3;
+
+    // after rhs
     refstr_t de;
     de.data = "de";
     de.len  = 2;
 
+    // after rhs
     refstr_t e;
     e.data = "e";
     e.len  = 1;
 
-    // [bb.*, dd.*]
+    // [bb, dd]
     str_range_t range;
     range.lhs = bb;
     range.rhs = dd;
@@ -147,6 +159,7 @@ TEST(str, range) {
     EXPECT_EQ(str_range_cmp(&range, &bb),  0);
     EXPECT_EQ(str_range_cmp(&range, &cc),  0);
     EXPECT_EQ(str_range_cmp(&range, &dd),  0);
+    EXPECT_EQ(str_range_cmp(&range, &ddd), 1);
     EXPECT_EQ(str_range_cmp(&range, &de),  1);
     EXPECT_EQ(str_range_cmp(&range, &e),   1);
 }
