add delimiter argument to gufi_query_distributed

Author: calccrypto

scripts/gufi_query_distributed

@@ -81,6 +81,7 @@ def subtree_cmd(args, filename, idx, target):
     cmd = [
         args.gufi_query,
         '-n', str(args.threads),
+        '-d', str(args.delim),
         '-y', str(args.level),
         '-D', filename,
     ]
@@ -96,6 +97,7 @@ def top_cmd(args, target):
     cmd = [
         args.gufi_query,
         '-n', str(args.threads),
+        '-d', str(args.delim),
         '-z', str(args.level - 1),
     ]
 
@@ -123,6 +125,10 @@ def parse_args():
                         type=gufi_common.get_positive,
                         default=1)
 
+    parser.add_argument('--delim',   '-d',  metavar='c',
+                        type=gufi_common.get_char,
+                        default=b'\x1e')
+
     for flag in SQL_FLAGS:
         parser.add_argument('-{0}'.format(flag), metavar='SQL', type=str)
 

test/regression/gufi_distributed.expected

@@ -16,7 +16,7 @@ Splitting 4 paths into 5 groups of max size 1
 Waiting for slurm jobs to complete
 
 # Query Index
-$ gufi_query_distributed --sbatch "sbatch" --gufi_query "gufi_query" --threads 2 slurm "hostfile" 1 "search2/prefix" -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;"
+$ gufi_query_distributed --sbatch "sbatch" --gufi_query "gufi_query" --threads 2 --delim "|" slurm "hostfile" 1 "search2/prefix" -S "SELECT rpath(sname, sroll), mtime FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;"
 Splitting 4 paths into 5 groups of max size 1
     Range 0: 1 path on localhost
         directory directory
@@ -37,29 +37,29 @@ cat the following slurm job output files to get complete results:
 
 # combine output files
 $ cat slurm.fake-0.out slurm.fake-1.out slurm.fake-2.out slurm.fake-3.out slurm.fake-4.out
-prefix2
-prefix2/.hidden
-prefix2/1KB
-prefix2/1MB
-prefix2/directory
-prefix2/directory/executable
-prefix2/directory/readonly
-prefix2/directory/subdirectory
-prefix2/directory/subdirectory/directory_symlink
-prefix2/directory/subdirectory/repeat_name
-prefix2/directory/writable
-prefix2/empty_directory
-prefix2/file_symlink
-prefix2/leaf_directory
-prefix2/leaf_directory/leaf_file1
-prefix2/leaf_directory/leaf_file2
-prefix2/old_file
-prefix2/repeat_name
-prefix2/unusual#? directory ,
-prefix2/unusual#? directory ,/unusual, name?#
+prefix2/.hidden|10
+prefix2/1KB|1024
+prefix2/1MB|1048576
+prefix2/directory/executable|1
+prefix2/directory/readonly|2
+prefix2/directory/subdirectory/directory_symlink|4
+prefix2/directory/subdirectory/repeat_name|5
+prefix2/directory/subdirectory|6
+prefix2/directory/writable|3
+prefix2/directory|7
+prefix2/empty_directory|8
+prefix2/file_symlink|9
+prefix2/leaf_directory/leaf_file1|11
+prefix2/leaf_directory/leaf_file2|12
+prefix2/leaf_directory|13
+prefix2/old_file|0
+prefix2/repeat_name|14
+prefix2/unusual#? directory ,/unusual, name?#|15
+prefix2/unusual#? directory ,|16
+prefix2|17
 
 # Diff original index results against the combined results
-$ diff <("gufi_query" -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort) <(cat slurm.fake-0.out slurm.fake-1.out slurm.fake-2.out slurm.fake-3.out slurm.fake-4.out | sort | sed 's/search2\/prefix/search\/prefix/g')
+$ diff <("gufi_query" -d "|" -S "SELECT rpath(sname, sroll), mtime FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;" "prefix" | sort) <(cat slurm.fake-0.out slurm.fake-1.out slurm.fake-2.out slurm.fake-3.out slurm.fake-4.out | sort | sed 's/search2\/prefix/search\/prefix/g')
 
 # Diff original index results against querying the new index from a single node
 $ diff <(gufi_query -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort) <(gufi_query -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "search2/prefix" | sort | sed 's/search2\/prefix/search\/prefix/g')
@@ -103,7 +103,7 @@ prefix/unusual#? directory ,
 prefix/unusual#? directory ,/unusual, name?#
 
 # Diff original index results against the trace files
-$ diff <(gufi_query -d " " -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort | sed 's/search\///g') <(cat traces.0.0 traces.1.0 traces.2.0 traces.3.0 traces.0 | awk -F'|' '{ print $1 }' | sort)
+$ diff <(gufi_query -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort | sed 's/search\///g') <(cat traces.0.0 traces.1.0 traces.2.0 traces.3.0 traces.0 | awk -F'|' '{ print $1 }' | sort)
 
 # Use existing group files (path_list.4 does not exist)
 $ gufi_dir2index_distributed --sbatch "sbatch" --gufi_dir2index "gufi_dir2index" --use-existing-group-files slurm "hostfile" 1 "prefix" "search2" 2>/dev/null
@@ -136,7 +136,7 @@ Splitting 4 paths into 5 groups of max size 1
 Waiting for ssh jobs to complete
 
 # Query Index
-$ gufi_query_distributed --ssh "ssh" --gufi_query "gufi_query" --threads 2 ssh "hostfile" 1 "search2/prefix" -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" --output_prefix "output"
+$ gufi_query_distributed --ssh "ssh" --gufi_query "gufi_query" --threads 2 --delim "|" ssh "hostfile" 1 "search2/prefix" -S "SELECT rpath(sname, sroll), mtime FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;" --output_prefix "output"
 Splitting 4 paths into 5 groups of max size 1
     Range 0: 1 path on localhost
         directory directory
@@ -152,29 +152,29 @@ cat "output.*" to get complete results
 
 # combine output files
 $ cat output.localhost.0.0 output.127.0.0.1.1.0 output.localhost.2.0 output.127.0.0.1.3.0 output.127.0.0.1.top.0 output.localhost.0.1 output.127.0.0.1.1.1 output.localhost.2.1 output.127.0.0.1.3.1 output.127.0.0.1.top.1
-prefix2
-prefix2/.hidden
-prefix2/1KB
-prefix2/1MB
-prefix2/directory
-prefix2/directory/executable
-prefix2/directory/readonly
-prefix2/directory/subdirectory
-prefix2/directory/subdirectory/directory_symlink
-prefix2/directory/subdirectory/repeat_name
-prefix2/directory/writable
-prefix2/empty_directory
-prefix2/file_symlink
-prefix2/leaf_directory
-prefix2/leaf_directory/leaf_file1
-prefix2/leaf_directory/leaf_file2
-prefix2/old_file
-prefix2/repeat_name
-prefix2/unusual#? directory ,
-prefix2/unusual#? directory ,/unusual, name?#
+prefix2/.hidden|10
+prefix2/1KB|1024
+prefix2/1MB|1048576
+prefix2/directory/executable|1
+prefix2/directory/readonly|2
+prefix2/directory/subdirectory/directory_symlink|4
+prefix2/directory/subdirectory/repeat_name|5
+prefix2/directory/subdirectory|6
+prefix2/directory/writable|3
+prefix2/directory|7
+prefix2/empty_directory|8
+prefix2/file_symlink|9
+prefix2/leaf_directory/leaf_file1|11
+prefix2/leaf_directory/leaf_file2|12
+prefix2/leaf_directory|13
+prefix2/old_file|0
+prefix2/repeat_name|14
+prefix2/unusual#? directory ,/unusual, name?#|15
+prefix2/unusual#? directory ,|16
+prefix2|17
 
 # Diff original index results against the combined results
-$ diff <("gufi_query" -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort) <(cat output.localhost.0.0 output.127.0.0.1.1.0 output.localhost.2.0 output.127.0.0.1.3.0 output.127.0.0.1.top.0 output.localhost.0.1 output.127.0.0.1.1.1 output.localhost.2.1 output.127.0.0.1.3.1 output.127.0.0.1.top.1 | sort | sed 's/search2\/prefix/search\/prefix/g')
+$ diff <("gufi_query" -d "|" -S "SELECT rpath(sname, sroll), mtime FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;" "prefix" | sort) <(cat output.localhost.0.0 output.127.0.0.1.1.0 output.localhost.2.0 output.127.0.0.1.3.0 output.127.0.0.1.top.0 output.localhost.0.1 output.127.0.0.1.1.1 output.localhost.2.1 output.127.0.0.1.3.1 output.127.0.0.1.top.1 | sort | sed 's/search2\/prefix/search\/prefix/g')
 
 # Diff original index results against querying the new index from a single node
 $ diff <(gufi_query -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort) <(gufi_query -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "search2/prefix" | sort | sed 's/search2\/prefix/search\/prefix/g')
@@ -218,7 +218,7 @@ prefix/unusual#? directory ,
 prefix/unusual#? directory ,/unusual, name?#
 
 # Diff original index results against the trace files
-$ diff <(gufi_query -d " " -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort | sed 's/search\///g') <(cat traces.0.0 traces.1.0 traces.2.0 traces.3.0 traces.0 | awk -F'|' '{ print $1 }' | sort)
+$ diff <(gufi_query -d "|" -S "SELECT rpath(sname, sroll) FROM vrsummary;" -E "SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;" "prefix" | sort | sed 's/search\///g') <(cat traces.0.0 traces.1.0 traces.2.0 traces.3.0 traces.0 | awk -F'|' '{ print $1 }' | sort)
 
 # Use existing group files (path_list.4 does not exist)
 $ gufi_dir2index_distributed --ssh "ssh" --gufi_dir2index "gufi_dir2index" --use-existing-group-files ssh "hostfile" 1 "prefix" "search2" 2>/dev/null | tail -n 9

test/regression/gufi_distributed.sh.in

@@ -149,7 +149,7 @@ trace_files() {
 cleanup() {
     # shellcheck disable=SC2207
     traces=($(trace_files))
-    rm -rf "${SEARCH2}" "${HOSTFILE}" "${groups[@]}" "${slurm_files[@]}" "${ssh_files[@]}" "${traces[@]}"
+    rm -rf "${SEARCH2}" "${HOSTFILE}" "${groups[@]}" "${slurm_files[@]}" "slurm.fake-5.out" "${ssh_files[@]}" "${traces[@]}"
 }
 
 cleanup_exit() {
@@ -182,15 +182,15 @@ run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --sbatch \"${SBATCH_FAKE}\" --gufi_di
 rm -rf "${slurm_files[@]}"
 
 echo "# Query Index"
-run_no_sort "${GUFI_QUERY_DISTRIBUTED} --sbatch \"${SBATCH_FAKE}\" --gufi_query \"${GUFI_QUERY}\" --threads ${THREADS} slurm \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${INDEXROOT2}\" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\""
+run_no_sort "${GUFI_QUERY_DISTRIBUTED} --sbatch \"${SBATCH_FAKE}\" --gufi_query \"${GUFI_QUERY}\" --threads ${THREADS} --delim \"${DELIM}\" slurm \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${INDEXROOT2}\" -S \"SELECT rpath(sname, sroll), mtime FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;\""
 
 echo "# combine output files"
 # shellcheck disable=SC2145
 run_sort "cat ${slurm_files[@]}" | @SED@ "s/${INDEXROOT2//\//\\/}/prefix2/g"
 
 echo "# Diff original index results against the combined results"
 # shellcheck disable=SC2145,SC2153
-run_no_sort "@DIFF@ <(\"${GUFI_QUERY}\" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort) <(cat ${slurm_files[@]} | sort | sed 's/${INDEXROOT2//\//\\/}/${INDEXROOT//\//\\/}/g')"
+run_no_sort "@DIFF@ <(\"${GUFI_QUERY}\" -d \"${DELIM}\" -S \"SELECT rpath(sname, sroll), mtime FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;\" \"${INDEXROOT}\" | sort) <(cat ${slurm_files[@]} | sort | sed 's/${INDEXROOT2//\//\\/}/${INDEXROOT//\//\\/}/g')"
 
 echo "# Diff original index results against querying the new index from a single node"
 run_no_sort "@DIFF@ <(${GUFI_QUERY} -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort) <(${GUFI_QUERY} -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT2}\" | sort | sed 's/${INDEXROOT2//\//\\/}/${INDEXROOT//\//\\/}/g')"
@@ -208,7 +208,7 @@ run_sort "cat ${traces[@]} | @AWK@ -F'${DELIM}' '{ print \$1 }'"
 
 echo "# Diff original index results against the trace files"
 # shellcheck disable=SC2145,SC2153
-run_no_sort "@DIFF@ <(${GUFI_QUERY} -d \" \" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort | @SED@ 's/${SEARCH//\//\\/}\\///g') <(cat ${traces[@]} | @AWK@ -F'${DELIM}' '{ print \$1 }' | sort)"
+run_no_sort "@DIFF@ <(${GUFI_QUERY} -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort | @SED@ 's/${SEARCH//\//\\/}\\///g') <(cat ${traces[@]} | @AWK@ -F'${DELIM}' '{ print \$1 }' | sort)"
 
 echo "# Use existing group files (path_list.4 does not exist)"
 run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --sbatch \"${SBATCH_FAKE}\" --gufi_dir2index \"${GUFI_DIR2INDEX}\" --use-existing-group-files slurm \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${SRCDIR}\" \"${SEARCH2}\" 2>/dev/null"
@@ -227,15 +227,15 @@ echo "# Index Source Tree"
 run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --ssh \"${SSH_FAKE}\" --gufi_dir2index \"${GUFI_DIR2INDEX}\" ssh \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${SRCDIR}\" \"${SEARCH2}\" 2>/dev/null"
 
 echo "# Query Index"
-run_no_sort "${GUFI_QUERY_DISTRIBUTED} --ssh \"${SSH_FAKE}\" --gufi_query \"${GUFI_QUERY}\" --threads ${THREADS} ssh \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${INDEXROOT2}\" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" --output_prefix \"output\""
+run_no_sort "${GUFI_QUERY_DISTRIBUTED} --ssh \"${SSH_FAKE}\" --gufi_query \"${GUFI_QUERY}\" --threads ${THREADS} --delim \"${DELIM}\" ssh \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${INDEXROOT2}\" -S \"SELECT rpath(sname, sroll), mtime FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;\" --output_prefix \"output\""
 
 echo "# combine output files"
 # shellcheck disable=SC2145
 run_sort "cat ${ssh_files[@]}" | @SED@ "s/${INDEXROOT2//\//\\/}/prefix2/g"
 
 echo "# Diff original index results against the combined results"
 # shellcheck disable=SC2145,SC2153
-run_no_sort "@DIFF@ <(\"${GUFI_QUERY}\" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort) <(cat ${ssh_files[@]} | sort | sed 's/${INDEXROOT2//\//\\/}/${INDEXROOT//\//\\/}/g')"
+run_no_sort "@DIFF@ <(\"${GUFI_QUERY}\" -d \"${DELIM}\" -S \"SELECT rpath(sname, sroll), mtime FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name, mtime FROM vrpentries;\" \"${INDEXROOT}\" | sort) <(cat ${ssh_files[@]} | sort | sed 's/${INDEXROOT2//\//\\/}/${INDEXROOT//\//\\/}/g')"
 
 echo "# Diff original index results against querying the new index from a single node"
 run_no_sort "@DIFF@ <(${GUFI_QUERY} -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort) <(${GUFI_QUERY} -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT2}\" | sort | sed 's/${INDEXROOT2//\//\\/}/${INDEXROOT//\//\\/}/g')"
@@ -251,7 +251,7 @@ run_sort "cat ${traces[@]} | @AWK@ -F'${DELIM}' '{ print \$1 }'"
 
 echo "# Diff original index results against the trace files"
 # shellcheck disable=SC2145,SC2153
-run_no_sort "@DIFF@ <(${GUFI_QUERY} -d \" \" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort | @SED@ 's/${SEARCH//\//\\/}\\///g') <(cat ${traces[@]} | @AWK@ -F'${DELIM}' '{ print \$1 }' | sort)"
+run_no_sort "@DIFF@ <(${GUFI_QUERY} -d \"${DELIM}\" -S \"SELECT rpath(sname, sroll) FROM vrsummary;\" -E \"SELECT rpath(sname, sroll) || '/' || name FROM vrpentries;\" \"${INDEXROOT}\" | sort | @SED@ 's/${SEARCH//\//\\/}\\///g') <(cat ${traces[@]} | @AWK@ -F'${DELIM}' '{ print \$1 }' | sort)"
 
 echo "# Use existing group files (path_list.4 does not exist)"
 run_no_sort "${GUFI_DIR2INDEX_DISTRIBUTED} --ssh \"${SSH_FAKE}\" --gufi_dir2index \"${GUFI_DIR2INDEX}\" --use-existing-group-files ssh \"${HOSTFILE}\" ${DISTRIBUTED_LEVEL} \"${SRCDIR}\" \"${SEARCH2}\" 2>/dev/null | tail -n 9"