miscellaneous fixes

fix parallel_cpr path checking
lustre_plugin depend on install_dependencies
wrap gufi_dirfd error check with #ifdef __APPLE__
gufi_dir2index test update
try_skip_lstat with DT_UNKNOWN test
blobop unit test updates

Author: calccrypto

contrib/plugins/CMakeLists.txt

@@ -65,5 +65,6 @@ find_path(LUSTRE_PLUGIN "lustre/lustreapi.h")
 # If so, build the lustre example plugin:
 if (LUSTRE_PLUGIN)
   add_library(lustre_plugin SHARED lustre_plugin.c)
+  add_dependencies(lustre_plugin install_dependencies)
   target_link_libraries(lustre_plugin lustreapi)
 endif()

src/parallel_cpr.c

@@ -278,6 +278,69 @@ static int cpr_dir(QPTPool_ctx_t *ctx, void *data) {
     return rc;
 }
 
+/*
+ * check if lhs contains rhs
+ *
+ * lhs and rhs should be absolute paths
+ *
+ * lhs and rhs are modified but should exit this function with their original values
+ * flip changes the order of the paths printed
+ *
+ * returns -1 on error
+ *          0 on lhs does not contain rhs
+ *          1 on lhs containes rhs
+ */
+static int check_contains(char *lhs, char *rhs, const int flip) {
+    struct stat lhs_st;
+    if (stat(lhs, &lhs_st) != 0) {
+        const int err = errno;
+        fprintf(stderr, "Error: Cannot stat \"%s\": %s (%d)\n",
+                lhs, strerror(err), err);
+        return -1;
+    }
+
+    /* last path segment is not followed by a slash, so have to handle separately */
+    struct stat rhs_st;
+    if (stat(rhs, &rhs_st) != 0) {
+        const int err = errno;
+        fprintf(stderr, "Error: Cannot stat \"%s\": %s (%d)\n",
+                rhs, strerror(err), err);
+        return -1;
+    }
+
+    if (lhs_st.st_ino == rhs_st.st_ino) {
+        fprintf(stderr, "Error: Not copying into itself: \"%s\" -> \"%s\"\n",
+                flip?rhs:lhs, flip?lhs:rhs);
+        return 1;
+    }
+
+    size_t len = trailing_match_index(rhs, strlen(rhs) - 1, "/", 1);
+    while (len > 1) {
+        rhs[len - 1] = '\0';
+
+        struct stat rhs_st;
+        const int stat_rc = stat(rhs, &rhs_st);
+        rhs[len - 1] = '/';
+
+        if (stat_rc != 0) {
+            const int err = errno;
+            fprintf(stderr, "Error: Cannot stat \"%s\": %s (%d)\n",
+                    rhs, strerror(err), err); /* rhs is the full path, not the stat-ed path */
+            return -1;
+        }
+
+        if (lhs_st.st_ino == rhs_st.st_ino) {
+            fprintf(stderr, "Error: Not copying into itself: \"%s\" -> \"%s\"\n",
+                    flip?rhs:lhs, flip?lhs:rhs);
+            return 1;
+        }
+
+        len = trailing_match_index(rhs, len - 1, "/", 1);
+    }
+
+    return 0;
+}
+
 static int setup(const str_t *dst) {
     const mode_t old_umask = umask(0);
     umask(old_umask); /* reset umask */
@@ -315,13 +378,12 @@ int main(int argc, char * argv[]) {
         return EXIT_FAILURE;
     }
 
-    /* get realpath after set up to ensure the path exists */
-    const char *dst = pa.dst.data;
-    const char *real_dst = realpath(dst, NULL);
+    /* get realpath after set up to ensure the destination path exists */
+    char *real_dst = realpath(pa.dst.data, NULL);
     if (!real_dst) {
         const int err = errno;
         fprintf(stderr, "Error: Cannot get realpath of \"%s\": %s (%d)\n",
-                dst, strerror(err), err);
+                pa.dst.data, strerror(err), err);
         input_fini(&pa.in);
         return EXIT_FAILURE;
     }
@@ -330,7 +392,7 @@ int main(int argc, char * argv[]) {
     if (QPTPool_start(ctx) != 0) {
         fprintf(stderr, "Error: Failed to start thread pool\n");
         QPTPool_destroy(ctx);
-        free((char *) real_dst);
+        free(real_dst);
         input_fini(&pa.in);
         return EXIT_FAILURE;
     }
@@ -359,13 +421,20 @@ int main(int argc, char * argv[]) {
         work->root_parent.len = dirname_len(path, work->name_len);
 
         if (S_ISDIR(st.st_mode)) {
-            const char *real_path = realpath(path, NULL);
-            const int match = !strncmp(real_dst, real_path, strlen(real_path));
-            free((char *) real_path);
-            if (match) {
-                fprintf(stderr, "Not copying into itself: \"%s\" -> \"%s\"\n",
-                        path, dst);
+            char *real_path = realpath(path, NULL);
+
+            /*
+             * check if current source is under the destination
+             * or if the destination is under the current source
+             */
+            const int contains = (check_contains(real_path, real_dst, 0) ||
+                                  check_contains(real_dst, real_path, 1));
+
+            free(real_path);
+
+            if (contains) {
                 rc = 1;
+                free(work);
                 continue;
             }
 
@@ -419,7 +488,7 @@ int main(int argc, char * argv[]) {
     const uint64_t threads_completed = QPTPool_threads_completed(ctx);
     QPTPool_destroy(ctx);
 
-    free((char *) real_dst);
+    free(real_dst);
 
     input_fini(&pa.in);
 

src/utils.c

@@ -829,6 +829,7 @@ ssize_t read_size(const int fd, void *buf, const size_t size) {
  */
 int gufi_dirfd(DIR *d) {
     int d_fd = dirfd(d);
+    #ifdef __APPLE__
     if (d_fd < 0) {
         /*
          * We should never get here. glibc's dirfd(3) never return errors, and
@@ -838,7 +839,7 @@ int gufi_dirfd(DIR *d) {
         fprintf(stderr, "BUG: dirfd(3) failed: errno = %d\n", errno);
         exit(1);
     }
-
+    #endif
     return d_fd;
 }
 

test/regression/gufi_dir2index.expected

@@ -376,6 +376,10 @@ Could not stat source directory "": No such file or directory (2)
 Total Dirs:          0
 Total Non-Dirs:      0
 
+# desintation path contains a file in a path segment
+$ gufi_dir2index -n 2 "prefix" "prefix/1KB/search"
+Could not create prefix/1KB/search
+
 # skip file
 $ gufi_dir2index -n 2 --skip-file "skip" "prefix" "search"
 "search" Already exists!

test/regression/gufi_dir2index.sh.in

@@ -218,6 +218,9 @@ run_no_sort "${GUFI_DIR2INDEX} -n ${THREADS} \"${SRCDIR}/1KB\" \"${SEARCH}\"" |
 echo "# empty source path"
 run_no_sort "${GUFI_DIR2INDEX} -n ${THREADS} \"\" \"${TRACE}\""
 
+echo "# desintation path contains a file in a path segment"
+run_no_sort "${GUFI_DIR2INDEX} -n ${THREADS} \"${SRCDIR}\" \"${SRCDIR}/1KB/${SEARCH}\""
+
 echo "# skip file"
 run_no_sort "${GUFI_DIR2INDEX} -n ${THREADS} --skip-file \"${SKIP}\" \"${SRCDIR}\" \"${SEARCH}\""
 

test/regression/parallel_cpr.expected

@@ -84,6 +84,9 @@ $ compare xattrs of "prefix/old_file"
 $ compare xattrs of "prefix/repeat_name"
 $ compare xattrs of "prefix/unusual#? directory ,"
 $ compare xattrs of "prefix/unusual#? directory ,/unusual, name?#"
+# Copy to similar name
+$ parallel_cpr "prefix"   "prefix2"
+
 $ rm -rf "copy"
 
 $ parallel_cpr    "prefix/leaf_directory/leaf_file1" "copy"
@@ -111,23 +114,27 @@ prefix/1KB
 
 $ rm -rf "copy"
 
-$ parallel_cpr "prefix"   "copy"
+# copy to existing directory
+$ parallel_cpr "prefix" "copy"
 
 $ rm -rf "copy"
 
-$ parallel_cpr "prefix"   "prefix"
-Not copying into itself: "prefix" -> "prefix"
+$ parallel_cpr "prefix" "prefix"
+Error: Not copying into itself: "prefix" -> "prefix"
+
+$ parallel_cpr "prefix" "prefix/directory"
+Error: Not copying into itself: "prefix" -> "prefix/directory"
 
-$ parallel_cpr "prefix"   "prefix/directory"
-Not copying into itself: "prefix" -> "prefix/directory"
+$ parallel_cpr "prefix/directory" "prefix"
+Error: Not copying into itself: "prefix/directory" -> "prefix"
 
 $ parallel_cpr "badtrace" "copy"
 Error: Cannot lstat "badtrace": No such file or directory (2)
 
-$ parallel_cpr /dev/null       "copy"
+$ parallel_cpr /dev/null "copy"
 Not copying "/dev/null"
 
-$ parallel_cpr "prefix"   "prefix/old_file"
+$ parallel_cpr "prefix" "prefix/old_file"
 Error: Cannot copy to "prefix/old_file": File exists (17)
 
 $ parallel_cpr -n 18446744073709551615 "prefix" "copy"

test/regression/parallel_cpr.sh.in

@@ -67,9 +67,10 @@ set -e
 OUTPUT="parallel_cpr.out"
 
 CPR_ROOT="copy"
+CPR_ROOT2="${SRCDIR}2"
 
 cleanup() {
-    rm -rf "${CPR_ROOT}"
+    rm -rf "${CPR_ROOT}" "${CPR_ROOT2}"
 }
 
 cleanup_exit() {
@@ -114,6 +115,9 @@ do
 done
 set -e
 
+echo "# Copy to similar name"
+run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\"   \"${CPR_ROOT2}\""
+
 # copy single files/links
 
 run_no_sort "rm -rf \"${CPR_ROOT}\""
@@ -128,26 +132,27 @@ run_no_sort "readlink \"${CPR_ROOT}/file_symlink\""
 
 run_no_sort "rm -rf \"${CPR_ROOT}\""
 
-# copy to existing directory
+echo "# copy to existing directory"
 mkdir "${CPR_ROOT}"
-run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\"   \"${CPR_ROOT}\""
+run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\" \"${CPR_ROOT}\""
 
 # errors
 
 run_no_sort "rm -rf \"${CPR_ROOT}\""
 
 # copy into self
-run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\"   \"${SRCDIR}\""
-run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\"   \"${SRCDIR}/directory\""
+run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\" \"${SRCDIR}\""
+run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\" \"${SRCDIR}/directory\""
+run_no_sort "${PARALLEL_CPR} \"${SRCDIR}/directory\" \"${SRCDIR}\""
 
 # non-existant source
 run_no_sort "${PARALLEL_CPR} \"${BADTRACE}\" \"${CPR_ROOT}\""
 
 # non-directory/link/file source
-run_no_sort "${PARALLEL_CPR} /dev/null       \"${CPR_ROOT}\""
+run_no_sort "${PARALLEL_CPR} /dev/null \"${CPR_ROOT}\""
 
 # directory to file
-run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\"   \"${SRCDIR}/old_file\""
+run_no_sort "${PARALLEL_CPR} \"${SRCDIR}\" \"${SRCDIR}/old_file\""
 
 # bad thread count
 run_no_sort "${PARALLEL_CPR} -n 18446744073709551615 \"${SRCDIR}\" \"${CPR_ROOT}\""

test/unit/googletest/addqueryfuncs.cpp

@@ -64,6 +64,7 @@ OF SUCH DAMAGE.
 
 #include <cmath>
 #include <ctime>
+#include <fstream>
 #include <grp.h>
 #include <pwd.h>
 #include <stdlib.h>
@@ -533,38 +534,72 @@ static int copy_blob_column_callback(void *args, int count, char **data, char **
 }
 
 TEST(addqueryfuncs, blobop) {
-    const char data[] = "abc def\n\x00\x01\x02 \x03\x04\x05\nghi jkl";
-    const std::size_t len = sizeof(data) - 1;
-
     // sqlite3 does not like binary in SQL statements, so write contents to temp file first
     char temp[] = "XXXXXX";
-    int fd = mkstemp(temp);
-    ASSERT_GT(fd, -1);
-    ASSERT_EQ(write(fd, data, len), (ssize_t) len);
-    close(fd);
+    {
+        const int fd = mkstemp(temp);
+        ASSERT_GT(fd, -1);
+        close(fd);
+    }
 
     sqlite3 *db = nullptr;
     ASSERT_EQ(sqlite3_open(SQLITE_MEMORY, &db), SQLITE_OK);
     ASSERT_NE(db, nullptr);
-
     ASSERT_EQ(addqueryfuncs(db), 0);
 
-    // have to insert into table to get length later without opening file twice
     char cmd[1024];
-    snprintf(cmd, sizeof(cmd), "CREATE TABLE data(col BLOB); INSERT INTO data SELECT blobop('cat %s');", temp);
-    ASSERT_EQ(sqlite3_exec(db, cmd, nullptr, nullptr, nullptr), SQLITE_OK);
-
     str_t output;
+    memset(&output, 0, sizeof(output));
     char *err = nullptr;
-    ASSERT_EQ(sqlite3_exec(db, "SELECT length(col), col FROM data;",
-                           copy_blob_column_callback, &output, &err), SQLITE_OK);
-    EXPECT_EQ(err, nullptr);
-    EXPECT_EQ(output.len, len);
-    ASSERT_NE(output.data, nullptr);
-    EXPECT_EQ(memcmp(output.data, data, len), 0);
-    str_free_existing(&output);
-    sqlite3_free(err);
-    err = nullptr;
+
+    // have to insert into table to get length later without opening file twice
+
+    // empty file
+    {
+        snprintf(cmd, sizeof(cmd), "CREATE TABLE data(col BLOB); INSERT INTO data SELECT blobop('cat %s');", temp);
+        ASSERT_EQ(sqlite3_exec(db, cmd, nullptr, nullptr, nullptr), SQLITE_OK);
+
+        ASSERT_EQ(sqlite3_exec(db, "SELECT length(col), col FROM data; DROP TABLE data;",
+                               copy_blob_column_callback, &output, &err), SQLITE_OK);
+        EXPECT_EQ(err, nullptr);
+        EXPECT_EQ(output.len, (std::size_t) 0);
+        ASSERT_NE(output.data, nullptr);
+        str_free_existing(&output);
+    }
+
+    // short result
+    {
+        const char data[] = "abc def\n\x00\x01\x02 \x03\x04\x05\nghi jkl";
+        const std::size_t len = sizeof(data) - 1;
+        std::ofstream(temp).write(data, len);
+        snprintf(cmd, sizeof(cmd), "CREATE TABLE data(col BLOB); INSERT INTO data SELECT blobop('cat %s');", temp);
+        ASSERT_EQ(sqlite3_exec(db, cmd, nullptr, nullptr, nullptr), SQLITE_OK);
+
+        ASSERT_EQ(sqlite3_exec(db, "SELECT length(col), col FROM data; DROP TABLE data;",
+                               copy_blob_column_callback, &output, &err), SQLITE_OK);
+        EXPECT_EQ(err, nullptr);
+        EXPECT_EQ(output.len, len);
+        ASSERT_NE(output.data, nullptr);
+        EXPECT_EQ(memcmp(output.data, data, len), 0);
+        str_free_existing(&output);
+    }
+
+    // "long" result (force a reallocation)
+    {
+        const std::size_t len = 256;
+        const std::string A(len, 'a');
+        std::ofstream(temp) << A;
+        snprintf(cmd, sizeof(cmd), "CREATE TABLE data(col BLOB); INSERT INTO data SELECT blobop('cat %s');", temp);
+        ASSERT_EQ(sqlite3_exec(db, cmd, NULL, NULL, NULL), SQLITE_OK);
+
+        ASSERT_EQ(sqlite3_exec(db, "SELECT length(col), col FROM data; DROP TABLE data;",
+                               copy_blob_column_callback, &output, &err), SQLITE_OK);
+        EXPECT_EQ(err, nullptr);
+        EXPECT_EQ(output.len, len);
+        ASSERT_NE(output.data, nullptr);
+        EXPECT_EQ(memcmp(output.data, A.c_str(), len), 0);
+        str_free_existing(&output);
+    }
 
     // not enough file descriptors
     {
@@ -575,7 +610,8 @@ TEST(addqueryfuncs, blobop) {
         fewer_fds.rlim_cur = 3;
         ASSERT_EQ(setrlimit(RLIMIT_NOFILE, &fewer_fds), 0);
 
-        ASSERT_EQ(sqlite3_exec(db, "SELECT blobop('echo blob');", copy_columns_callback, &output, &err), SQLITE_ERROR);
+        ASSERT_EQ(sqlite3_exec(db, "SELECT blobop('echo blob');",
+                               copy_columns_callback, &output, &err), SQLITE_ERROR);
         EXPECT_NE(err, nullptr);
         sqlite3_free(err);
         err = nullptr;