unify external database list processing

also reorganized some includes

Author: calccrypto

include/external.h

@@ -66,10 +66,10 @@ OF SUCH DAMAGE.
 #define EXTERNAL_H
 
 #include <stdlib.h>
+#include <sqlite3.h>
 
 #include "SinglyLinkedList.h"
 #include "bf.h"
-#include "dbutils.h"
 #include "trie.h"
 
 #ifdef __cplusplus
@@ -113,18 +113,21 @@ extern const refstr_t EXTERNAL_TYPE_XATTR;               /* convenience struct *
 #define EXTERNAL_TYPE_USER_DB_LEN  (sizeof(EXTERNAL_TYPE_USER_DB_NAME) - 1)
 extern const refstr_t EXTERNAL_TYPE_USER_DB;             /* convenience struct */
 
-#define EXTERNAL_ATTACH_PREFIX "extdb"
+#define EXTERNAL_ATTACH_PREFIX     "extdb"
 
 int create_external_tables(const char *name, sqlite3 *db, void *args);
 
-size_t external_create_query(char *sql,         const size_t sql_size,
-                             const char *cols,  const size_t cols_len,
-                             const char *table, const size_t table_len,
-                             const refstr_t *type,
-                             const refstr_t *extra);
-
 int external_insert(sqlite3 *db, const char *type, const long long int pinode, const char *filename);
 
+typedef int (*external_process_db_f)(struct input *in, void *args,
+                                     const long long int pinode,
+                                     const char *filename);
+
+size_t external_read_file(struct input *in,
+                          struct work *child,
+                          external_process_db_f func,
+                          void *args);
+
 /*
  * external_concatenate
  *

include/gufi_query/timers.h

@@ -71,7 +71,7 @@ OF SUCH DAMAGE.
 
 #include "OutputBuffers.h"
 #include "SinglyLinkedList.h"
-#include "debug.h"
+#include "../debug.h"
 
 /* ************************************************************************* */
 /* global/pool variables */
@@ -150,19 +150,19 @@ enum {
  */
 #if defined(DEBUG) && (defined(CUMULATIVE_TIMES) || defined(PER_THREAD_STATS))
 #define descend_timestamp_start(dts, name)                      \
-    struct start_end *name = malloc(sizeof(struct start_end));  \
-    clock_gettime(CLOCK_MONOTONIC, &(name)->start);             \
-                                                                \
-    do {                                                        \
-        /* get list where new timestamp will be placed */       \
-        sll_t *dst = &((dts)[dts_##name]);                      \
-                                                                \
-        /* keep track of new timestamp */                       \
-        sll_push(dst, name);                                    \
-    } while (0)
+//     struct start_end *name = malloc(sizeof(struct start_end));  \
+//     clock_gettime(CLOCK_MONOTONIC, &(name)->start);             \
+//                                                                 \
+//     do {                                                        \
+//         /* get list where new timestamp will be placed */       \
+//         sll_t *dst = &((dts)[dts_##name]);                      \
+//                                                                 \
+//         /* keep track of new timestamp */                       \
+//         sll_push(dst, name);                                    \
+//     } while (0)
 
 #define descend_timestamp_end(name)                             \
-    clock_gettime(CLOCK_MONOTONIC, &(name)->end);
+//     clock_gettime(CLOCK_MONOTONIC, &(name)->end);
 #else
 #define descend_timestamp_start(dts, name)
 #define descend_timestamp_end(name)
@@ -205,11 +205,11 @@ void timestamps_init(timestamps_t *ts, struct timespec *zero);
 
 #if defined(DEBUG) && (defined(CUMULATIVE_TIMES) || defined(PER_THREAD_STATS))
 #define thread_timestamp_start(tts, name)           \
-    struct start_end *name = &tts[tts_##name];      \
-    clock_gettime(CLOCK_MONOTONIC, &(name)->start);
+//     struct start_end *name = &tts[tts_##name];      \
+//     clock_gettime(CLOCK_MONOTONIC, &(name)->start);
 
 #define thread_timestamp_end(name)                  \
-    clock_gettime(CLOCK_MONOTONIC, &(name)->end);
+//     clock_gettime(CLOCK_MONOTONIC, &(name)->end);
 #else
 #define thread_timestamp_start(tts, name)
 #define thread_timestamp_end(name)

include/utils.h

@@ -72,6 +72,7 @@ OF SUCH DAMAGE.
 #include "QueuePerThreadPool.h"
 #include "bf.h"
 #include "config.h"
+#include "external.h"
 #include "trie.h"
 
 #ifdef __cplusplus
@@ -106,10 +107,6 @@ int shortpath(const char *name, char *nameout, char *endname);
 /* descend */
 typedef int (*process_nondir_f)(struct work *nondir, struct entry_data *ed, void *nondir_args);
 
-/* returns number of external databases tracked */
-typedef size_t (*process_external_db_f)(struct input *in,
-                                        struct work *child, void *args);
-
 struct descend_counters {
     size_t dirs;
     size_t dirs_insitu;
@@ -126,7 +123,7 @@ int descend(QPTPool_t *ctx, const size_t id, void *args,
             struct input *in, struct work *work, ino_t inode,
             DIR *dir, trie_t *skip, const int skip_db, const int stat_entries,
             QPTPoolFunc_t processdir, process_nondir_f processnondir, void *nondir_args,
-            process_external_db_f process_external_db, void *external_db_args,
+            external_process_db_f process_external_db, void *external_db_args,
             struct descend_counters *counters);
 
 /* ******************************************************* */

src/external.c

@@ -62,10 +62,13 @@ OF SUCH DAMAGE.
 
 
 
+#include <errno.h>
+#include <fcntl.h>
 #include <string.h>
+#include <unistd.h>
 
+#include "dbutils.h"
 #include "external.h"
-#include "utils.h"
 
 /*
  * attachname, mode, uid, and gid are not strictly necessary but can be useful
@@ -108,11 +111,11 @@ int create_external_tables(const char *name, sqlite3 *db, void *args) {
 }
 
 /* SELECT <cols> FROM <table> [WHERE (type == '<type>') AND [extra]]; */
-size_t external_create_query(char *sql,         const size_t sql_size,
-                             const char *cols,  const size_t cols_len,
-                             const char *table, const size_t table_len,
-                             const refstr_t *type,
-                             const refstr_t *extra) {
+static size_t external_create_query(char *sql,         const size_t sql_size,
+                                    const char *cols,  const size_t cols_len,
+                                    const char *table, const size_t table_len,
+                                    const refstr_t *type,
+                                    const refstr_t *extra) {
     size_t len = SNFORMAT_S(sql, sql_size, 4,
                             "SELECT ", (size_t) 7,
                             cols, cols_len,
@@ -166,6 +169,90 @@ int external_insert(sqlite3 *db, const char *type, const long long int pinode, c
     return 0;
 }
 
+/* returns 1 for good/not checked, 0 for error */
+static int check_is_db(const int check_extdb_valid, const char *path) {
+    int rc = !check_extdb_valid;
+
+    if (check_extdb_valid) {
+        /* open the path to make sure it eventually resolves to a file */
+        sqlite3 *extdb = opendb(path, SQLITE_OPEN_READONLY, 0, 0, NULL, NULL);
+        if (extdb) {
+            /* make sure this file is a sqlite3 db */
+            /* can probably skip this check */
+            char *err = NULL;
+            if (sqlite3_exec(extdb, "SELECT '' FROM sqlite_master;", NULL, NULL, &err) == SQLITE_OK) {
+                rc = 1;
+            }
+            else {
+                fprintf(stderr, "Warning: %s is not a db: %s\n",
+                        path, err);
+                sqlite3_free(err);
+            }
+
+            closedb(extdb);
+        }
+    }
+
+    return rc;
+}
+
+size_t external_read_file(struct input *in,
+                          struct work *child,
+                          external_process_db_f func,
+                          void *args) {
+    int extdb_list = open(child->name, O_RDONLY);
+    if (extdb_list < 0) {
+        const int err = errno;
+        fprintf(stderr, "Error: Could not open user external database list in %s: %s (%d)\n",
+                child->name, strerror(err), err);
+        return 0;
+    }
+
+    size_t rc = 0;
+
+    char *line = NULL;
+    size_t len = 0;
+    off_t offset = 0;
+    while (getline_fd(&line, &len, extdb_list, &offset, 512) > 0) {
+        char extdb_path_stack[MAXPATH];
+        char *extdb_path = line;
+
+        /* resolve relative paths */
+        if (line[0] != '/')  {
+            char path[MAXPATH];
+            SNFORMAT_S(path, sizeof(path), 2,
+                       child->name, child->name_len - child->basename_len,
+                       /* basename does not include slash, so don't need to add another one */
+                       line, len);
+
+            if (!realpath(path, extdb_path_stack)) {
+                const int err = errno;
+                fprintf(stderr, "Error: Could not resolve external database path %s: %s (%d)\n",
+                        path, strerror(err), err);
+                free(line);
+                line = NULL;
+                continue;
+            }
+
+            extdb_path = extdb_path_stack;
+        }
+
+        if (check_is_db(in->check_extdb_valid, extdb_path) == 1){
+            rc += !func(in, args, child->pinode, extdb_path);
+        }
+
+        free(line);
+        line = NULL;
+    }
+
+    free(line);
+    line = NULL;
+
+    close(extdb_list);
+
+    return rc;
+}
+
 int external_concatenate(sqlite3 *db,
                          const refstr_t *type,
                          const refstr_t *extra,

src/gufi_dir2index.c

@@ -63,7 +63,6 @@ OF SUCH DAMAGE.
 
 
 #include <errno.h>
-#include <fcntl.h>
 #include <dirent.h>
 #include <inttypes.h>
 #include <stdlib.h>
@@ -154,80 +153,14 @@ static int process_nondir(struct work *entry, struct entry_data *ed, void *args)
     return 0;
 }
 
-static size_t track_external(struct input *in,
-                             struct work *child,
-                             void *args) {
-    sqlite3 *db = (sqlite3 *) args;
-
-    int extdb_list = open(child->name, O_RDONLY);
-    if (extdb_list < 0) {
-        const int err = errno;
-        fprintf(stderr, "Error: Could not open user external database list in %s: %s (%d)\n",
-                child->name, strerror(err), err);
-        return 0;
-    }
-
-    size_t rc = 0;
-
-    char *line = NULL;
-    size_t len = 0;
-    off_t offset = 0;
-    while (getline_fd(&line, &len, extdb_list, &offset, 512) > 0) {
-        char extdb_path_stack[MAXPATH];
-        char *extdb_path = line;
-
-        /* resolve relative paths */
-        if (line[0] != '/')  {
-            char path[MAXPATH];
-            SNFORMAT_S(path, sizeof(path), 2,
-                       child->name, child->name_len - child->basename_len,
-                       /* basename does not include slash, so don't need to add another one */
-                       line, len);
-
-            if (!realpath(path, extdb_path_stack)) {
-                const int err = errno;
-                fprintf(stderr, "Error: Could not resolve external database path %s: %s (%d)\n",
-                        path, strerror(err), err);
-                free(line);
-                line = NULL;
-                continue;
-            }
-
-            extdb_path = extdb_path_stack;
-        }
+static int process_external(struct input *in, void *args,
+                            const long long int pinode,
+                            const char *filename) {
+    (void) in;
 
-        if (in->check_extdb_valid) {
-            /* open the path to make sure it eventually resolves to a file */
-            sqlite3 *extdb = opendb(extdb_path, SQLITE_OPEN_READONLY, 0, 0, NULL, NULL);
-            char *err = NULL;
-
-            /* make sure this file is a sqlite3 db */
-            /* can probably skip this check */
-            if (sqlite3_exec(extdb, "SELECT '' FROM sqlite_master;", NULL, NULL, &err) == SQLITE_OK) {
-                rc += !external_insert(db, EXTERNAL_TYPE_USER_DB.data, child->pinode, extdb_path);
-            }
-            else {
-                fprintf(stderr, "Warning: Not tracking requested external db: %s: %s\n",
-                        extdb_path, err);
-                sqlite3_free(err);
-            }
-
-            closedb(extdb);
-        }
-        else {
-            rc += !external_insert(db, EXTERNAL_TYPE_USER_DB.data, child->pinode, extdb_path);
-        }
-
-        free(line);
-        line = NULL;
-    }
-
-    free(line);
-    line = NULL;
-
-    close(extdb_list);
+    sqlite3 *db = (sqlite3 *) args;
 
-    return rc;
+    return external_insert(db, EXTERNAL_TYPE_USER_DB_NAME, pinode, filename);
 }
 
 static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
@@ -318,7 +251,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
     descend(ctx, id, pa, in, nda.work, nda.ed.statuso.st_ino, dir,
             in->skip, 0, 0,
             processdir, process_nondir, &nda,
-            track_external, nda.db,
+            process_external, nda.db,
             &ctrs);
     stopdb(nda.db);