per-directory external database list

indexer (probably admin) no longer has to keep track of external database basenames that users want to track
directory owner can create a file called external.gufi listing all external db files to track
changed -q to trigger validating external db files before tracking them
    defaults to not set

Author: calccrypto

docs/latex/sections/external_databases.tex

@@ -63,37 +63,29 @@
 \subsection{External Databases}
 In addition to extended attributes, users may place SQLite3 database
 files into the source file system to be indexed and attached into GUFI
-for querying\footnote{An alternative method might be to allow for
-users to place simple text files of known names into the source
-filesystem for automatic indexing. This would remove the requirement
-that the administrator for the index knows what external database file
-names exist.}. This effectively allows for users to associate and
+for querying. This effectively allows for users to associate and
 query aribitrary data with filesystem metadata.
 
 The extended attribute code uses the more generic external database
 infrastructure to track the extended attributes that were not rolled
 in.
 
 \subsubsection{Usage}
-In order to index a set of external database files that serve the same
-purpose, they should have the same basename that will be discovered
-during indexing by passing \texttt{-q} to \gufidirindex or
-\gufidirtrace.
+All external database files for a set of data should have the same
+basename. The files should be listed in a simple text file called
+``external.gufi'' that is in the directory where a subset of the data
+will be joined with GUFI data. If the path is relative, it will be
+concatentated with the parent directory's path to produce an absolute
+path.
 
-For example, if SQLite3 files with the name ``external.db" are placed
-into various directories in the source filesystem, the call to index
-the source tree should look like:
+\texttt{-q} may be passed in during indexing in order to verify that
+an entry listed in ``external.gufi'' is indeed a database file.
 
-\hspace*{\fill} \gufidirindex \texttt{-q "external.db" "ext" ...} \hspace*{\fill}
+One more more additional database files, referred to as the
+``schema template", or ``template", containing a copy of the same
+schema as each external database table should be maintained somewhere
+for usage when querying. The template file should generally have empty
+tables, but nothing prevents non-empty tables from being provided.
 
-which will result in each ``external.db" file being attached with the
-name ``ext".
-
-An additional database file, referred to as the ``schema template", or
-``template", containing a copy of the ``external.db" schema should be
-maintained somewhere for usage when querying. This template file
-should generally have empty tables, but nothing prevents non-empty
-files from being provided.
-
-Note that the external databases will also be included as an entry in
-the index (this may change in the future).
+Note that ``external.gufi'' will also be included as an entry in the
+index (this may change in the future).

include/bf.h

@@ -238,9 +238,8 @@ struct input {
    /* compress work items (if compression library was found) */
    int compress;
 
-   /* used when indexing (-q) */
-   trie_t *map_external;      /* full path -> attach name */
-   size_t map_external_count; /* only keeps track of unique full paths */
+   /* check if a listed external db is valid when indexing (-q) */
+   int check_extdb_valid;
 
    /* used when querying (-Q) */
    sll_t external_attach;     /* list of eus_t */

include/external.h

@@ -76,6 +76,10 @@ OF SUCH DAMAGE.
 extern "C" {
 #endif
 
+/* name of file that user can create to list external databases to track */
+#define EXTERNAL_DB_USER_FILE      "external.gufi"
+#define EXTERNAL_DB_USER_FILE_LEN  (sizeof(EXTERNAL_DB_USER_FILE) - 1)
+
 /*
  *
  * these tables/views only exist in db.db and list
@@ -85,21 +89,26 @@ extern "C" {
  * matter the usage of the external databases
  */
 
+/* table */
 #define EXTERNAL_DBS_PWD           "external_dbs_pwd"    /* *.db files found during indexing */
 extern const char EXTERNAL_DBS_PWD_CREATE[];
 extern const char EXTERNAL_DBS_PWD_INSERT[];
 
+/* table */
 #define EXTERNAL_DBS_ROLLUP        "external_dbs_rollup" /* *.db files brought up during rollup */
 extern const char EXTERNAL_DBS_ROLLUP_CREATE[];
 extern const char EXTERNAL_DBS_ROLLUP_INSERT[];
 
+/* view */
 #define EXTERNAL_DBS               "external_dbs"
 #define EXTERNAL_DBS_LEN           (sizeof(EXTERNAL_DBS) - 1)
 
+/* column value */
 #define EXTERNAL_TYPE_XATTR_NAME   "xattrs"
 #define EXTERNAL_TYPE_XATTR_LEN    (sizeof(EXTERNAL_TYPE_XATTR_NAME) - 1)
 extern const refstr_t EXTERNAL_TYPE_XATTR;               /* convenience struct */
 
+/* column value */
 #define EXTERNAL_TYPE_USER_DB_NAME "user_db"
 #define EXTERNAL_TYPE_USER_DB_LEN  (sizeof(EXTERNAL_TYPE_USER_DB_NAME) - 1)
 extern const refstr_t EXTERNAL_TYPE_USER_DB;             /* convenience struct */

include/utils.h

@@ -106,9 +106,9 @@ int shortpath(const char *name, char *nameout, char *endname);
 /* descend */
 typedef int (*process_nondir_f)(struct work *nondir, struct entry_data *ed, void *nondir_args);
 
-/* returns 1 for success, 0 for failure */
-typedef int (*process_external_db_f)(struct input *in,
-                                     struct work *child, void *args);
+/* returns number of external databases tracked */
+typedef size_t (*process_external_db_f)(struct input *in,
+                                        struct work *child, void *args);
 
 struct descend_counters {
     size_t dirs;

src/bf.c

@@ -108,8 +108,6 @@ struct input *input_init(struct input *in) {
         trie_insert(in->skip, ".",  1, NULL, NULL);
         trie_insert(in->skip, "..", 2, NULL, NULL);
 
-        in->map_external            = trie_alloc();
-
         sll_init(&in->external_attach);
     }
 
@@ -119,7 +117,6 @@ struct input *input_init(struct input *in) {
 void input_fini(struct input *in) {
     if (in) {
         sll_destroy(&in->external_attach, free);
-        trie_free(in->map_external);
         trie_free(in->skip);
     }
 }
@@ -181,7 +178,7 @@ void print_help(const char* prog_name,
       case 'M': printf("  -M <bytes>             target memory footprint"); break;
       case 'C': printf("  -C <count>             Number of subdirectories allowed to be enqueued for parallel processing. Any remainders will be processed in-situ"); break;
       case 'e': printf("  -e                     compress work items"); break;
-      case 'q': printf("  -q <basename>          Basename of file to keep track of during indexing"); break;
+      case 'q': printf("  -q                     check that external databases are valid before tracking during indexing"); break;
       case 'Q': printf("  -Q <basename>\n"
                        "     <table>\n"
                        "     <template>.<table>\n"
@@ -240,7 +237,7 @@ void show_input(struct input* in, int retval) {
    printf("in.target_memory_footprint  = %" PRIu64 "\n",   in->target_memory_footprint);
    printf("in.subdir_limit             = %zu\n",           in->subdir_limit);
    printf("in.compress                 = %d\n",            in->compress);
-   printf("in.external_db_count        = %zu\n",           in->map_external_count);
+   printf("in.check_extdb_valid        = %d\n",            in->check_extdb_valid);
    size_t i = 0;
    sll_loop(&in->external_attach, node) {
        eus_t *eus = (eus_t *) sll_node_data(node);
@@ -483,14 +480,8 @@ int parse_cmd_line(int         argc,
           in->compress = 1;
           break;
 
-      case 'q':               // file basename -> attach name
-          {
-              refstr_t basename;
-              INSTALL_STR(&basename, optarg);
-
-              in->map_external_count += !trie_search(in->map_external, basename.data, basename.len, NULL);
-              trie_insert(in->map_external, basename.data, basename.len, NULL, NULL);
-          }
+      case 'q':
+          in->check_extdb_valid = 1;
           break;
 
       case 'Q':

src/gufi_dir2index.c

@@ -63,6 +63,7 @@ OF SUCH DAMAGE.
 
 
 #include <errno.h>
+#include <fcntl.h>
 #include <dirent.h>
 #include <inttypes.h>
 #include <stdlib.h>
@@ -153,35 +154,78 @@ static int process_nondir(struct work *entry, struct entry_data *ed, void *args)
     return 0;
 }
 
-static int track_external(struct input *in,
-                          struct work *child,
-                          void *args) {
+static size_t track_external(struct input *in,
+                             struct work *child,
+                             void *args) {
     sqlite3 *db = (sqlite3 *) args;
 
-    refstr_t *ext = NULL;
-    if (!trie_search(in->map_external, child->name + child->name_len - child->basename_len,
-                     child->basename_len, (void **) &ext)) {
+    int extdb_list = open(child->name, O_RDONLY);
+    if (extdb_list < 0) {
+        const int err = errno;
+        fprintf(stderr, "Error: Could not open user external database list in %s: %s (%d)\n",
+                child->name, strerror(err), err);
         return 0;
     }
 
-    int rc = 1;
+    size_t rc = 0;
+
+    char *line = NULL;
+    size_t len = 0;
+    off_t offset = 0;
+    while (getline_fd(&line, &len, extdb_list, &offset, 512) > 0) {
+        char extdb_path_stack[MAXPATH];
+        char *extdb_path = line;
+
+        /* resolve relative paths */
+        if (line[0] != '/')  {
+            char path[MAXPATH];
+            SNFORMAT_S(path, sizeof(path), 2,
+                       child->name, child->name_len - child->basename_len,
+                       /* basename does not include slash, so don't need to add another one */
+                       line, len);
+
+            if (!realpath(path, extdb_path_stack)) {
+                const int err = errno;
+                fprintf(stderr, "Error: Could not resolve external database path %s: %s (%d)\n",
+                        path, strerror(err), err);
+                free(line);
+                line = NULL;
+                continue;
+            }
+
+            extdb_path = extdb_path_stack;
+        }
 
-    /* open the path to make sure it eventually resolves to a file */
-    sqlite3 *extdb = opendb(child->name, SQLITE_OPEN_READONLY, 0, 0, NULL, NULL);
-    char *err = NULL;
+        if (in->check_extdb_valid) {
+            /* open the path to make sure it eventually resolves to a file */
+            sqlite3 *extdb = opendb(extdb_path, SQLITE_OPEN_READONLY, 0, 0, NULL, NULL);
+            char *err = NULL;
+
+            /* make sure this file is a sqlite3 db */
+            /* can probably skip this check */
+            if (sqlite3_exec(extdb, "SELECT '' FROM sqlite_master;", NULL, NULL, &err) == SQLITE_OK) {
+                rc += !external_insert(db, EXTERNAL_TYPE_USER_DB.data, child->pinode, extdb_path);
+            }
+            else {
+                fprintf(stderr, "Warning: Not tracking requested external db: %s: %s\n",
+                        extdb_path, err);
+                sqlite3_free(err);
+            }
+
+            closedb(extdb);
+        }
+        else {
+            rc += !external_insert(db, EXTERNAL_TYPE_USER_DB.data, child->pinode, extdb_path);
+        }
 
-    /* make sure this file is a sqlite3 db */
-    /* can probably skip this check */
-    if (sqlite3_exec(extdb, "SELECT '' FROM sqlite_master;", NULL, NULL, &err) == SQLITE_OK) {
-        rc = !external_insert(db, EXTERNAL_TYPE_USER_DB.data, child->pinode, child->name);
+        free(line);
+        line = NULL;
     }
-    else {
-        fprintf(stderr, "Warning: Not tracking requested external db: %s: %s\n",
-                child->name, err);
-        sqlite3_free(err);
-        rc = 0;
-    }
-    closedb(extdb);
+
+    free(line);
+    line = NULL;
+
+    close(extdb_list);
 
     return rc;
 }
@@ -399,7 +443,7 @@ static void sub_help(void) {
 
 int main(int argc, char *argv[]) {
     struct PoolArgs pa;
-    int idx = parse_cmd_line(argc, argv, "hHn:xz:k:M:C:" COMPRESS_OPT "q:", 2, "input_dir... output_dir", &pa.in);
+    int idx = parse_cmd_line(argc, argv, "hHn:xz:k:M:C:" COMPRESS_OPT "q", 2, "input_dir... output_dir", &pa.in);
     if (pa.in.helped)
         sub_help();
     if (idx < 0) {